2024-11-16 14:51:08,099 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 14:51:08,112 main DEBUG Took 0.011188 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 14:51:08,113 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 14:51:08,113 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 14:51:08,114 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 14:51:08,115 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,123 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 14:51:08,137 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,139 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,139 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,140 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,142 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,143 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,145 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,146 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,147 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,148 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,149 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,149 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,150 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,151 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:51:08,152 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,152 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 14:51:08,154 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:51:08,156 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 14:51:08,158 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 14:51:08,159 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 14:51:08,161 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 14:51:08,161 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 14:51:08,169 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 14:51:08,171 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 14:51:08,173 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 14:51:08,173 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 14:51:08,174 main DEBUG createAppenders(={Console}) 2024-11-16 14:51:08,175 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-16 14:51:08,175 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 14:51:08,175 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-16 14:51:08,176 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 14:51:08,176 main DEBUG OutputStream closed 2024-11-16 14:51:08,176 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 14:51:08,177 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 14:51:08,177 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-16 14:51:08,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 14:51:08,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 14:51:08,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 14:51:08,258 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 14:51:08,259 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 14:51:08,260 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 14:51:08,261 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 14:51:08,261 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 14:51:08,262 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 14:51:08,262 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 14:51:08,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 14:51:08,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 14:51:08,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 14:51:08,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 14:51:08,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 14:51:08,265 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 14:51:08,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 14:51:08,267 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 14:51:08,271 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 14:51:08,271 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-16 14:51:08,272 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 14:51:08,273 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-16T14:51:08,555 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954 2024-11-16 14:51:08,557 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 14:51:08,558 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T14:51:08,565 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-16T14:51:08,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T14:51:08,602 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/cluster_f34ac9e9-6867-0b4c-ab87-b17e81b33f1e, deleteOnExit=true 2024-11-16T14:51:08,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T14:51:08,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/test.cache.data in system properties and HBase conf 2024-11-16T14:51:08,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T14:51:08,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/hadoop.log.dir in system properties and HBase conf 2024-11-16T14:51:08,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T14:51:08,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T14:51:08,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T14:51:08,693 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T14:51:08,780 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T14:51:08,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T14:51:08,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T14:51:08,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T14:51:08,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T14:51:08,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T14:51:08,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T14:51:08,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T14:51:08,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T14:51:08,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T14:51:08,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/nfs.dump.dir in system properties and HBase conf 2024-11-16T14:51:08,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/java.io.tmpdir in system properties and HBase conf 2024-11-16T14:51:08,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T14:51:08,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T14:51:08,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T14:51:09,762 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T14:51:09,834 INFO [Time-limited test {}] log.Log(170): Logging initialized @2418ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T14:51:09,899 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:51:09,950 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:51:09,970 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:51:09,970 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:51:09,971 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T14:51:09,981 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:51:09,984 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:51:09,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:51:10,147 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/java.io.tmpdir/jetty-localhost-35427-hadoop-hdfs-3_4_1-tests_jar-_-any-5922886553133968661/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T14:51:10,152 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:35427} 2024-11-16T14:51:10,153 INFO [Time-limited test {}] server.Server(415): Started @2737ms 2024-11-16T14:51:10,649 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:51:10,656 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:51:10,657 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:51:10,657 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:51:10,658 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T14:51:10,658 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:51:10,659 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:51:10,760 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/java.io.tmpdir/jetty-localhost-42287-hadoop-hdfs-3_4_1-tests_jar-_-any-16893429296314611505/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:51:10,760 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:42287} 2024-11-16T14:51:10,761 INFO [Time-limited test {}] server.Server(415): Started @3345ms 2024-11-16T14:51:10,808 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T14:51:11,649 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/cluster_f34ac9e9-6867-0b4c-ab87-b17e81b33f1e/data/data2/current/BP-443048117-172.17.0.2-1731768669338/current, will proceed with Du for space computation calculation, 2024-11-16T14:51:11,649 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/cluster_f34ac9e9-6867-0b4c-ab87-b17e81b33f1e/data/data1/current/BP-443048117-172.17.0.2-1731768669338/current, will proceed with Du for space computation calculation, 2024-11-16T14:51:11,674 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T14:51:11,714 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954 2024-11-16T14:51:11,722 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x79899268c6caaf3 with lease ID 0x2359d6a992c0dc5: Processing first storage report for DS-44c7984b-200b-4b18-a3fd-c071718bd24f from datanode DatanodeRegistration(127.0.0.1:46343, datanodeUuid=12639581-9049-4216-a9f7-1e0789fcc1d5, infoPort=39391, infoSecurePort=0, ipcPort=36961, storageInfo=lv=-57;cid=testClusterID;nsid=2111138531;c=1731768669338) 2024-11-16T14:51:11,723 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x79899268c6caaf3 with lease ID 0x2359d6a992c0dc5: from storage DS-44c7984b-200b-4b18-a3fd-c071718bd24f node DatanodeRegistration(127.0.0.1:46343, datanodeUuid=12639581-9049-4216-a9f7-1e0789fcc1d5, infoPort=39391, infoSecurePort=0, ipcPort=36961, storageInfo=lv=-57;cid=testClusterID;nsid=2111138531;c=1731768669338), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T14:51:11,723 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x79899268c6caaf3 with lease ID 0x2359d6a992c0dc5: Processing first storage report for DS-0028d562-d980-4dd3-a152-a7029f1ea006 from datanode DatanodeRegistration(127.0.0.1:46343, datanodeUuid=12639581-9049-4216-a9f7-1e0789fcc1d5, infoPort=39391, infoSecurePort=0, ipcPort=36961, storageInfo=lv=-57;cid=testClusterID;nsid=2111138531;c=1731768669338) 2024-11-16T14:51:11,723 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x79899268c6caaf3 with lease ID 0x2359d6a992c0dc5: from storage DS-0028d562-d980-4dd3-a152-a7029f1ea006 node DatanodeRegistration(127.0.0.1:46343, datanodeUuid=12639581-9049-4216-a9f7-1e0789fcc1d5, infoPort=39391, infoSecurePort=0, ipcPort=36961, storageInfo=lv=-57;cid=testClusterID;nsid=2111138531;c=1731768669338), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T14:51:11,796 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/cluster_f34ac9e9-6867-0b4c-ab87-b17e81b33f1e/zookeeper_0, clientPort=54092, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/cluster_f34ac9e9-6867-0b4c-ab87-b17e81b33f1e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/cluster_f34ac9e9-6867-0b4c-ab87-b17e81b33f1e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T14:51:11,806 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54092 2024-11-16T14:51:11,816 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:51:11,819 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:51:12,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741825_1001 (size=7) 2024-11-16T14:51:12,427 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d with version=8 2024-11-16T14:51:12,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/hbase-staging 2024-11-16T14:51:12,519 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T14:51:12,761 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fc3fda29f660:0 server-side Connection retries=45 2024-11-16T14:51:12,769 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:51:12,770 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:51:12,774 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:51:12,774 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:51:12,775 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:51:12,904 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T14:51:12,967 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T14:51:12,979 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T14:51:12,984 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:51:13,016 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 86781 (auto-detected) 2024-11-16T14:51:13,017 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T14:51:13,041 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35823 2024-11-16T14:51:13,066 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35823 connecting to ZooKeeper ensemble=127.0.0.1:54092 2024-11-16T14:51:13,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358230x0, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:51:13,173 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35823-0x101456a70e90000 connected 2024-11-16T14:51:13,275 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:51:13,280 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:51:13,290 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:51:13,294 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d, hbase.cluster.distributed=false 2024-11-16T14:51:13,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:51:13,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35823 2024-11-16T14:51:13,319 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35823 2024-11-16T14:51:13,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35823 2024-11-16T14:51:13,326 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35823 2024-11-16T14:51:13,326 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35823 2024-11-16T14:51:13,417 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fc3fda29f660:0 server-side Connection retries=45 2024-11-16T14:51:13,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:51:13,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:51:13,419 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:51:13,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:51:13,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:51:13,422 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T14:51:13,424 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:51:13,425 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43955 2024-11-16T14:51:13,427 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43955 connecting to ZooKeeper ensemble=127.0.0.1:54092 2024-11-16T14:51:13,428 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:51:13,431 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:51:13,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:439550x0, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:51:13,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:439550x0, quorum=127.0.0.1:54092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:51:13,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43955-0x101456a70e90001 connected 2024-11-16T14:51:13,459 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T14:51:13,465 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T14:51:13,467 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T14:51:13,472 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:51:13,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43955 2024-11-16T14:51:13,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43955 2024-11-16T14:51:13,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43955 2024-11-16T14:51:13,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43955 2024-11-16T14:51:13,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43955 2024-11-16T14:51:13,488 DEBUG [M:0;fc3fda29f660:35823 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fc3fda29f660:35823 2024-11-16T14:51:13,489 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fc3fda29f660,35823,1731768672605 2024-11-16T14:51:13,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:51:13,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:51:13,503 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fc3fda29f660,35823,1731768672605 2024-11-16T14:51:13,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:13,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T14:51:13,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:13,534 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T14:51:13,535 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fc3fda29f660,35823,1731768672605 from backup master directory 2024-11-16T14:51:13,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fc3fda29f660,35823,1731768672605 2024-11-16T14:51:13,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:51:13,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:51:13,547 WARN [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:51:13,547 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fc3fda29f660,35823,1731768672605 2024-11-16T14:51:13,550 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T14:51:13,552 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T14:51:13,603 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/hbase.id] with ID: ddbf7b10-fc3f-45cb-abf3-345a89badfe3 2024-11-16T14:51:13,603 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/.tmp/hbase.id 2024-11-16T14:51:13,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741826_1002 (size=42) 2024-11-16T14:51:14,014 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/.tmp/hbase.id]:[hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/hbase.id] 2024-11-16T14:51:14,059 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:51:14,065 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T14:51:14,084 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-16T14:51:14,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:14,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:14,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741827_1003 (size=196) 2024-11-16T14:51:14,576 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:51:14,579 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T14:51:14,592 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:14,595 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:51:14,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741828_1004 (size=1189) 2024-11-16T14:51:15,049 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store 2024-11-16T14:51:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741829_1005 (size=34) 2024-11-16T14:51:15,480 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T14:51:15,484 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:51:15,484 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T14:51:15,485 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:51:15,485 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:51:15,486 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T14:51:15,486 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:51:15,487 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:51:15,488 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731768675484Disabling compacts and flushes for region at 1731768675484Disabling writes for close at 1731768675486 (+2 ms)Writing region close event to WAL at 1731768675486Closed at 1731768675486 2024-11-16T14:51:15,490 WARN [master/fc3fda29f660:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/.initializing 2024-11-16T14:51:15,490 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/WALs/fc3fda29f660,35823,1731768672605 2024-11-16T14:51:15,498 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:51:15,512 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc3fda29f660%2C35823%2C1731768672605, suffix=, logDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/WALs/fc3fda29f660,35823,1731768672605, archiveDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/oldWALs, maxLogs=10 2024-11-16T14:51:15,532 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/WALs/fc3fda29f660,35823,1731768672605/fc3fda29f660%2C35823%2C1731768672605.1731768675516, exclude list is [], retry=0 2024-11-16T14:51:15,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-44c7984b-200b-4b18-a3fd-c071718bd24f,DISK] 2024-11-16T14:51:15,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T14:51:15,582 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/WALs/fc3fda29f660,35823,1731768672605/fc3fda29f660%2C35823%2C1731768672605.1731768675516 2024-11-16T14:51:15,583 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39391:39391)] 2024-11-16T14:51:15,583 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:51:15,583 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:51:15,586 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,587 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T14:51:15,645 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:15,647 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:51:15,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T14:51:15,651 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:15,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:15,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T14:51:15,655 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:15,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:15,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T14:51:15,658 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:15,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:15,660 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,663 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,664 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,669 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,670 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,673 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T14:51:15,677 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:51:15,681 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:51:15,682 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64307939, jitterRate=-0.04173703491687775}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T14:51:15,689 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731768675598Initializing all the Stores at 1731768675600 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731768675601 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768675601Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768675602 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768675602Cleaning up temporary data from old regions at 1731768675670 (+68 ms)Region opened successfully at 1731768675688 (+18 ms) 2024-11-16T14:51:15,690 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T14:51:15,719 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209c1685, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fc3fda29f660/172.17.0.2:0 2024-11-16T14:51:15,745 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T14:51:15,755 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T14:51:15,755 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T14:51:15,758 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T14:51:15,759 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T14:51:15,764 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T14:51:15,764 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T14:51:15,787 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T14:51:15,794 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T14:51:15,841 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T14:51:15,844 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T14:51:15,846 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T14:51:15,857 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T14:51:15,861 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T14:51:15,866 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T14:51:15,874 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T14:51:15,875 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T14:51:15,882 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T14:51:15,901 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T14:51:15,912 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T14:51:15,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:51:15,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:51:15,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:15,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:15,927 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fc3fda29f660,35823,1731768672605, sessionid=0x101456a70e90000, setting cluster-up flag (Was=false) 2024-11-16T14:51:15,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:15,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:15,982 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T14:51:15,986 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fc3fda29f660,35823,1731768672605 2024-11-16T14:51:16,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:16,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:16,041 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T14:51:16,045 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fc3fda29f660,35823,1731768672605 2024-11-16T14:51:16,056 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T14:51:16,080 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(746): ClusterId : ddbf7b10-fc3f-45cb-abf3-345a89badfe3 2024-11-16T14:51:16,082 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T14:51:16,093 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T14:51:16,093 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T14:51:16,106 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T14:51:16,106 DEBUG [RS:0;fc3fda29f660:43955 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17b91b11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fc3fda29f660/172.17.0.2:0 2024-11-16T14:51:16,120 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fc3fda29f660:43955 2024-11-16T14:51:16,122 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T14:51:16,123 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T14:51:16,123 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T14:51:16,124 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T14:51:16,126 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(2659): reportForDuty to master=fc3fda29f660,35823,1731768672605 with port=43955, startcode=1731768673386 2024-11-16T14:51:16,131 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T14:51:16,136 DEBUG [RS:0;fc3fda29f660:43955 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T14:51:16,137 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T14:51:16,142 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fc3fda29f660,35823,1731768672605 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T14:51:16,148 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fc3fda29f660:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:51:16,148 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fc3fda29f660:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:51:16,148 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fc3fda29f660:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:51:16,149 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fc3fda29f660:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:51:16,149 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fc3fda29f660:0, corePoolSize=10, maxPoolSize=10 2024-11-16T14:51:16,149 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,149 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fc3fda29f660:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:51:16,149 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,155 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:51:16,157 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T14:51:16,157 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731768706157 2024-11-16T14:51:16,158 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T14:51:16,160 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T14:51:16,163 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:16,164 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T14:51:16,164 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T14:51:16,164 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T14:51:16,165 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T14:51:16,165 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T14:51:16,168 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40055, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T14:51:16,165 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,171 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T14:51:16,172 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T14:51:16,173 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T14:51:16,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741831_1007 (size=1321) 2024-11-16T14:51:16,175 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T14:51:16,175 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T14:51:16,177 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T14:51:16,177 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35823 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:16,177 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d 2024-11-16T14:51:16,177 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.large.0-1731768676176,5,FailOnTimeoutGroup] 2024-11-16T14:51:16,178 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.small.0-1731768676178,5,FailOnTimeoutGroup] 2024-11-16T14:51:16,178 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,178 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T14:51:16,180 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,180 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741832_1008 (size=32) 2024-11-16T14:51:16,203 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T14:51:16,203 WARN [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T14:51:16,305 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(2659): reportForDuty to master=fc3fda29f660,35823,1731768672605 with port=43955, startcode=1731768673386 2024-11-16T14:51:16,310 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35823 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fc3fda29f660,43955,1731768673386 2024-11-16T14:51:16,315 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35823 {}] master.ServerManager(517): Registering regionserver=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:16,324 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d 2024-11-16T14:51:16,325 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38773 2024-11-16T14:51:16,325 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T14:51:16,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T14:51:16,359 DEBUG [RS:0;fc3fda29f660:43955 {}] zookeeper.ZKUtil(111): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fc3fda29f660,43955,1731768673386 2024-11-16T14:51:16,359 WARN [RS:0;fc3fda29f660:43955 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:51:16,359 INFO [RS:0;fc3fda29f660:43955 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:51:16,360 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/WALs/fc3fda29f660,43955,1731768673386 2024-11-16T14:51:16,363 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fc3fda29f660,43955,1731768673386] 2024-11-16T14:51:16,389 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T14:51:16,404 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T14:51:16,409 INFO [RS:0;fc3fda29f660:43955 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T14:51:16,410 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,411 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T14:51:16,416 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T14:51:16,417 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,417 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,418 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,418 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,418 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,418 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,418 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:51:16,418 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,418 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,419 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,419 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,419 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,419 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:51:16,419 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fc3fda29f660:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:51:16,419 DEBUG [RS:0;fc3fda29f660:43955 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:51:16,420 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,420 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,420 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,421 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,421 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,421 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,43955,1731768673386-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:51:16,438 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T14:51:16,440 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,43955,1731768673386-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,440 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,440 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.Replication(171): fc3fda29f660,43955,1731768673386 started 2024-11-16T14:51:16,456 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:16,457 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1482): Serving as fc3fda29f660,43955,1731768673386, RpcServer on fc3fda29f660/172.17.0.2:43955, sessionid=0x101456a70e90001 2024-11-16T14:51:16,457 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T14:51:16,457 DEBUG [RS:0;fc3fda29f660:43955 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fc3fda29f660,43955,1731768673386 2024-11-16T14:51:16,458 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fc3fda29f660,43955,1731768673386' 2024-11-16T14:51:16,458 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T14:51:16,459 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T14:51:16,459 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T14:51:16,459 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T14:51:16,460 DEBUG [RS:0;fc3fda29f660:43955 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fc3fda29f660,43955,1731768673386 2024-11-16T14:51:16,460 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fc3fda29f660,43955,1731768673386' 2024-11-16T14:51:16,460 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T14:51:16,460 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T14:51:16,461 DEBUG [RS:0;fc3fda29f660:43955 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T14:51:16,461 INFO [RS:0;fc3fda29f660:43955 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T14:51:16,461 INFO [RS:0;fc3fda29f660:43955 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T14:51:16,568 INFO [RS:0;fc3fda29f660:43955 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:51:16,572 INFO [RS:0;fc3fda29f660:43955 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc3fda29f660%2C43955%2C1731768673386, suffix=, logDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/WALs/fc3fda29f660,43955,1731768673386, archiveDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/oldWALs, maxLogs=32 2024-11-16T14:51:16,588 DEBUG [RS:0;fc3fda29f660:43955 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/WALs/fc3fda29f660,43955,1731768673386/fc3fda29f660%2C43955%2C1731768673386.1731768676576, exclude list is [], retry=0 2024-11-16T14:51:16,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:51:16,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-44c7984b-200b-4b18-a3fd-c071718bd24f,DISK] 2024-11-16T14:51:16,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T14:51:16,595 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T14:51:16,595 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:16,596 INFO [RS:0;fc3fda29f660:43955 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/WALs/fc3fda29f660,43955,1731768673386/fc3fda29f660%2C43955%2C1731768673386.1731768676576 2024-11-16T14:51:16,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:51:16,596 DEBUG [RS:0;fc3fda29f660:43955 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39391:39391)] 2024-11-16T14:51:16,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T14:51:16,599 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T14:51:16,599 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:16,600 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:51:16,600 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T14:51:16,603 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T14:51:16,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:16,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:51:16,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T14:51:16,608 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T14:51:16,608 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:16,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:51:16,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T14:51:16,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740 2024-11-16T14:51:16,611 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740 2024-11-16T14:51:16,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T14:51:16,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T14:51:16,616 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:51:16,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T14:51:16,622 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:51:16,623 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65997915, jitterRate=-0.01655443012714386}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:51:16,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731768676590Initializing all the Stores at 1731768676592 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731768676592Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731768676592Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768676592Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731768676592Cleaning up temporary data from old regions at 1731768676615 (+23 ms)Region opened successfully at 1731768676626 (+11 ms) 2024-11-16T14:51:16,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T14:51:16,626 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T14:51:16,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T14:51:16,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T14:51:16,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T14:51:16,628 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T14:51:16,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731768676626Disabling compacts and flushes for region at 1731768676626Disabling writes for close at 1731768676626Writing region close event to WAL at 1731768676627 (+1 ms)Closed at 1731768676628 (+1 ms) 2024-11-16T14:51:16,632 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:51:16,632 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T14:51:16,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T14:51:16,645 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T14:51:16,648 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T14:51:16,804 DEBUG [fc3fda29f660:35823 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T14:51:16,821 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:16,828 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,43955,1731768673386, state=OPENING 2024-11-16T14:51:16,891 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T14:51:16,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:16,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:51:16,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:51:16,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:51:16,903 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T14:51:16,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:51:17,082 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T14:51:17,086 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60273, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T14:51:17,097 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T14:51:17,097 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:51:17,098 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T14:51:17,101 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc3fda29f660%2C43955%2C1731768673386.meta, suffix=.meta, logDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/WALs/fc3fda29f660,43955,1731768673386, archiveDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/oldWALs, maxLogs=32 2024-11-16T14:51:17,114 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/WALs/fc3fda29f660,43955,1731768673386/fc3fda29f660%2C43955%2C1731768673386.meta.1731768677102.meta, exclude list is [], retry=0 2024-11-16T14:51:17,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-44c7984b-200b-4b18-a3fd-c071718bd24f,DISK] 2024-11-16T14:51:17,121 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/WALs/fc3fda29f660,43955,1731768673386/fc3fda29f660%2C43955%2C1731768673386.meta.1731768677102.meta 2024-11-16T14:51:17,121 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39391:39391)] 2024-11-16T14:51:17,121 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:51:17,124 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T14:51:17,127 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T14:51:17,131 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T14:51:17,135 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T14:51:17,136 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:51:17,136 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T14:51:17,136 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T14:51:17,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T14:51:17,140 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T14:51:17,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:17,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:51:17,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T14:51:17,143 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T14:51:17,143 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:17,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:51:17,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T14:51:17,145 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T14:51:17,145 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:17,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:51:17,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T14:51:17,147 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T14:51:17,147 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:17,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:51:17,148 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T14:51:17,149 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740 2024-11-16T14:51:17,152 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740 2024-11-16T14:51:17,154 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T14:51:17,154 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T14:51:17,155 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:51:17,157 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T14:51:17,159 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72353372, jitterRate=0.07814925909042358}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:51:17,159 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T14:51:17,161 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731768677137Writing region info on filesystem at 1731768677137Initializing all the Stores at 1731768677139 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731768677139Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731768677139Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768677139Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731768677139Cleaning up temporary data from old regions at 1731768677154 (+15 ms)Running coprocessor post-open hooks at 1731768677159 (+5 ms)Region opened successfully at 1731768677160 (+1 ms) 2024-11-16T14:51:17,167 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731768677074 2024-11-16T14:51:17,177 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T14:51:17,177 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T14:51:17,179 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:17,181 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,43955,1731768673386, state=OPEN 2024-11-16T14:51:17,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:51:17,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:51:17,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:51:17,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:51:17,240 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:17,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T14:51:17,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,43955,1731768673386 in 336 msec 2024-11-16T14:51:17,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T14:51:17,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 613 msec 2024-11-16T14:51:17,258 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:51:17,258 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T14:51:17,277 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:17,278 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:17,295 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:17,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53585, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:17,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2320 sec 2024-11-16T14:51:17,317 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731768677317, completionTime=-1 2024-11-16T14:51:17,320 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-16T14:51:17,320 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T14:51:17,372 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-16T14:51:17,373 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731768737372 2024-11-16T14:51:17,373 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731768797373 2024-11-16T14:51:17,373 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 53 msec 2024-11-16T14:51:17,375 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,35823,1731768672605-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:17,376 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,35823,1731768672605-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:17,376 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,35823,1731768672605-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:17,378 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fc3fda29f660:35823, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:17,378 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:17,379 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:17,387 DEBUG [master/fc3fda29f660:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T14:51:17,405 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.857sec 2024-11-16T14:51:17,406 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T14:51:17,407 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T14:51:17,408 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T14:51:17,409 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T14:51:17,409 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T14:51:17,410 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,35823,1731768672605-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:51:17,411 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,35823,1731768672605-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T14:51:17,419 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T14:51:17,420 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T14:51:17,420 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,35823,1731768672605-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:51:17,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d57a2b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:17,494 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T14:51:17,494 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T14:51:17,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:17,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:17,506 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e930ca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:17,513 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:17,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7fbb0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:17,517 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:17,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:17,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fc3fda29f660,35823,1731768672605 2024-11-16T14:51:17,558 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=231, ProcessCount=11, AvailableMemoryMB=5524 2024-11-16T14:51:17,566 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:51:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fc3fda29f660,35823,1731768672605 2024-11-16T14:51:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a65e75c 2024-11-16T14:51:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:51:17,576 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:51:17,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:51:17,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:51:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:51:17,592 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:51:17,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-16T14:51:17,595 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:17,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:51:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:51:17,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741835_1011 (size=1061) 2024-11-16T14:51:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:51:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:51:18,014 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d 2024-11-16T14:51:18,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741836_1012 (size=53) 2024-11-16T14:51:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:51:18,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:51:18,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c410da0e7ea4faea21a8b41f311130b9, disabling compactions & flushes 2024-11-16T14:51:18,426 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:18,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:18,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. after waiting 0 ms 2024-11-16T14:51:18,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:18,426 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:18,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c410da0e7ea4faea21a8b41f311130b9: Waiting for close lock at 1731768678426Disabling compacts and flushes for region at 1731768678426Disabling writes for close at 1731768678426Writing region close event to WAL at 1731768678426Closed at 1731768678426 2024-11-16T14:51:18,428 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:51:18,434 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731768678428"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731768678428"}]},"ts":"1731768678428"} 2024-11-16T14:51:18,438 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:51:18,440 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:51:18,443 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768678440"}]},"ts":"1731768678440"} 2024-11-16T14:51:18,447 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:51:18,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c410da0e7ea4faea21a8b41f311130b9, ASSIGN}] 2024-11-16T14:51:18,452 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c410da0e7ea4faea21a8b41f311130b9, ASSIGN 2024-11-16T14:51:18,454 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c410da0e7ea4faea21a8b41f311130b9, ASSIGN; state=OFFLINE, location=fc3fda29f660,43955,1731768673386; forceNewPlan=false, retain=false 2024-11-16T14:51:18,607 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c410da0e7ea4faea21a8b41f311130b9, regionState=OPENING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:18,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c410da0e7ea4faea21a8b41f311130b9, ASSIGN because future has completed 2024-11-16T14:51:18,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:51:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:51:18,782 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:18,782 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:51:18,783 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,783 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:51:18,783 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,783 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,786 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,790 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:51:18,790 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c410da0e7ea4faea21a8b41f311130b9 columnFamilyName A 2024-11-16T14:51:18,791 DEBUG [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:18,792 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] regionserver.HStore(327): Store=c410da0e7ea4faea21a8b41f311130b9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:18,792 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,794 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:51:18,794 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c410da0e7ea4faea21a8b41f311130b9 columnFamilyName B 2024-11-16T14:51:18,794 DEBUG [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:18,795 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] regionserver.HStore(327): Store=c410da0e7ea4faea21a8b41f311130b9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:18,795 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,797 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:51:18,798 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c410da0e7ea4faea21a8b41f311130b9 columnFamilyName C 2024-11-16T14:51:18,798 DEBUG [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:18,799 INFO [StoreOpener-c410da0e7ea4faea21a8b41f311130b9-1 {}] regionserver.HStore(327): Store=c410da0e7ea4faea21a8b41f311130b9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:18,799 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:18,799 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,801 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,801 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,804 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,804 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,805 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:51:18,807 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,810 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:51:18,811 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c410da0e7ea4faea21a8b41f311130b9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74899839, jitterRate=0.11609457433223724}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:51:18,811 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:18,812 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c410da0e7ea4faea21a8b41f311130b9: Running coprocessor pre-open hook at 1731768678784Writing region info on filesystem at 1731768678784Initializing all the Stores at 1731768678786 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768678786Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768678786Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768678786Cleaning up temporary data from old regions at 1731768678804 (+18 ms)Running coprocessor post-open hooks at 1731768678811 (+7 ms)Region opened successfully at 1731768678812 (+1 ms) 2024-11-16T14:51:18,814 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., pid=6, masterSystemTime=1731768678772 2024-11-16T14:51:18,818 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:18,818 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:18,819 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c410da0e7ea4faea21a8b41f311130b9, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:18,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:51:18,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T14:51:18,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 in 209 msec 2024-11-16T14:51:18,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T14:51:18,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c410da0e7ea4faea21a8b41f311130b9, ASSIGN in 382 msec 2024-11-16T14:51:18,837 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:51:18,838 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768678838"}]},"ts":"1731768678838"} 2024-11-16T14:51:18,841 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:51:18,844 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:51:18,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2600 sec 2024-11-16T14:51:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:51:19,752 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:19,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1487a66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:19,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:19,760 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:19,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675bb8fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:19,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:19,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:19,764 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a0f579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,766 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:19,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:19,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:19,773 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:51:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fc3fda29f660,35823,1731768672605 2024-11-16T14:51:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6deb0a6f 2024-11-16T14:51:19,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:51:19,780 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:51:19,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d3f2f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:19,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:19,787 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ac9955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:19,790 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230faddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ad9e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:19,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:19,795 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3c5a0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:19,798 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e61ecee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6308d33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:19,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:19,803 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2523f4b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:19,807 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8014a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a822d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:19,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:19,812 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666f68d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:19,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:19,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:19,816 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d31b62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7813eeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:19,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:19,821 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@931f7e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:19,824 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3efb018f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f66d26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:19,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:19,829 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dcece4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:19,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b9095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@225d1960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:19,835 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:19,836 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e082570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:19,839 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:19,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce7111b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93b4e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,842 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:19,842 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:19,843 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b3fb7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:19,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:19,846 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:19,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62078f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7449cb3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:19,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:19,850 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a9e52e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:19,852 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b67e89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:19,857 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:19,857 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:19,860 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:19,860 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:19,861 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:19,861 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:19,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:19,863 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:19,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:19,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:19,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:19,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:19,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:19,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:19,865 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:19,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:19,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:19,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:19,867 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:19,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:19,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:19,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:51:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:51:19,872 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:51:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:19,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:19,877 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-16T14:51:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:51:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:51:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:19,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:19,883 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:51:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:51:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:51:19,889 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:19,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:51:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:51:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:19,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:51:19,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:19,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:19,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:19,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:19,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:19,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:19,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:51:20,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:20,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:20,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768740084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/9f4a21c6272c4d20b7fa084b13348fd0 is 50, key is test_row_0/A:col10/1731768679939/Put/seqid=0 2024-11-16T14:51:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768740079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741837_1013 (size=14341) 2024-11-16T14:51:20,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/9f4a21c6272c4d20b7fa084b13348fd0 2024-11-16T14:51:20,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768740092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768740097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768740102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:51:20,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768740222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768740222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/ff426543b83b4c29bc5d8bbcf2779bde is 50, key is test_row_0/B:col10/1731768679939/Put/seqid=0 2024-11-16T14:51:20,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768740242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768740242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768740243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741838_1014 (size=12001) 2024-11-16T14:51:20,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:20,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768740431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768740441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768740452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768740465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768740462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:51:20,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:20,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:20,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/ff426543b83b4c29bc5d8bbcf2779bde 2024-11-16T14:51:20,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/f4e29bfb3b21416d9099ffac8de0d8ed is 50, key is test_row_0/C:col10/1731768679939/Put/seqid=0 2024-11-16T14:51:20,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741839_1015 (size=12001) 2024-11-16T14:51:20,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/f4e29bfb3b21416d9099ffac8de0d8ed 2024-11-16T14:51:20,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/9f4a21c6272c4d20b7fa084b13348fd0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/9f4a21c6272c4d20b7fa084b13348fd0 2024-11-16T14:51:20,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:20,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:20,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768740742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/9f4a21c6272c4d20b7fa084b13348fd0, entries=200, sequenceid=13, filesize=14.0 K 2024-11-16T14:51:20,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768740752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/ff426543b83b4c29bc5d8bbcf2779bde as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/ff426543b83b4c29bc5d8bbcf2779bde 2024-11-16T14:51:20,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/ff426543b83b4c29bc5d8bbcf2779bde, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T14:51:20,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/f4e29bfb3b21416d9099ffac8de0d8ed as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f4e29bfb3b21416d9099ffac8de0d8ed 2024-11-16T14:51:20,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f4e29bfb3b21416d9099ffac8de0d8ed, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T14:51:20,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c410da0e7ea4faea21a8b41f311130b9 in 853ms, sequenceid=13, compaction requested=false 2024-11-16T14:51:20,812 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T14:51:20,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:20,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-16T14:51:20,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:20,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:20,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/b34512986b794b4fb352fc865ed073be is 50, key is test_row_0/A:col10/1731768680828/Put/seqid=0 2024-11-16T14:51:20,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741840_1016 (size=14341) 2024-11-16T14:51:20,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:20,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:20,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:20,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/b34512986b794b4fb352fc865ed073be 2024-11-16T14:51:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:20,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/8509cadbced54269bc11ea4eff6aaf24 is 50, key is test_row_0/B:col10/1731768680828/Put/seqid=0 2024-11-16T14:51:20,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768740912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741841_1017 (size=12001) 2024-11-16T14:51:20,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/8509cadbced54269bc11ea4eff6aaf24 2024-11-16T14:51:20,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/4ce8fe50bb7e4a88a45c9a44e9d3a311 is 50, key is test_row_0/C:col10/1731768680828/Put/seqid=0 2024-11-16T14:51:20,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768740941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768740947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:20,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741842_1018 (size=12001) 2024-11-16T14:51:20,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/4ce8fe50bb7e4a88a45c9a44e9d3a311 2024-11-16T14:51:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:51:21,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/b34512986b794b4fb352fc865ed073be as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b34512986b794b4fb352fc865ed073be 2024-11-16T14:51:21,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b34512986b794b4fb352fc865ed073be, entries=200, sequenceid=36, filesize=14.0 K 2024-11-16T14:51:21,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/8509cadbced54269bc11ea4eff6aaf24 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/8509cadbced54269bc11ea4eff6aaf24 2024-11-16T14:51:21,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/8509cadbced54269bc11ea4eff6aaf24, entries=150, sequenceid=36, filesize=11.7 K 2024-11-16T14:51:21,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/4ce8fe50bb7e4a88a45c9a44e9d3a311 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4ce8fe50bb7e4a88a45c9a44e9d3a311 2024-11-16T14:51:21,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4ce8fe50bb7e4a88a45c9a44e9d3a311, entries=150, sequenceid=36, filesize=11.7 K 2024-11-16T14:51:21,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=46.96 KB/48090 for c410da0e7ea4faea21a8b41f311130b9 in 251ms, sequenceid=36, compaction requested=false 2024-11-16T14:51:21,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:21,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:51:21,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:21,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:21,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:21,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:21,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:21,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:21,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/138192596162420684f9840c18764218 is 50, key is test_row_0/A:col10/1731768681084/Put/seqid=0 2024-11-16T14:51:21,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741843_1019 (size=16681) 2024-11-16T14:51:21,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768741261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768681238, max timestamp=1731768681268Num uniques 859; 2024-11-16T14:51:21,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:51:21,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768681238, max timestamp=1731768681268Num uniques 859; 2024-11-16T14:51:21,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:51:21,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768681238, max timestamp=1731768681268Num uniques 859; 2024-11-16T14:51:21,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:51:21,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768741343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768741345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768741345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768741345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:21,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:21,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768741472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768741472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768741482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768741482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:21,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:21,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/138192596162420684f9840c18764218 2024-11-16T14:51:21,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/c526034fa3dc4b90b534346dae7af218 is 50, key is test_row_0/B:col10/1731768681084/Put/seqid=0 2024-11-16T14:51:21,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741844_1020 (size=12001) 2024-11-16T14:51:21,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768741682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768741682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:21,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768741691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768741701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:21,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:21,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:21,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768741991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768741992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:22,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768742001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:22,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:22,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:22,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:22,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:22,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:22,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:22,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768742011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:51:22,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/c526034fa3dc4b90b534346dae7af218 2024-11-16T14:51:22,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/8942cb50b97d4495902a479ffb78e0c2 is 50, key is test_row_0/C:col10/1731768681084/Put/seqid=0 2024-11-16T14:51:22,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741845_1021 (size=12001) 2024-11-16T14:51:22,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/8942cb50b97d4495902a479ffb78e0c2 2024-11-16T14:51:22,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/138192596162420684f9840c18764218 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/138192596162420684f9840c18764218 2024-11-16T14:51:22,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/138192596162420684f9840c18764218, entries=250, sequenceid=48, filesize=16.3 K 2024-11-16T14:51:22,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/c526034fa3dc4b90b534346dae7af218 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/c526034fa3dc4b90b534346dae7af218 2024-11-16T14:51:22,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/c526034fa3dc4b90b534346dae7af218, entries=150, sequenceid=48, filesize=11.7 K 2024-11-16T14:51:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/8942cb50b97d4495902a479ffb78e0c2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8942cb50b97d4495902a479ffb78e0c2 2024-11-16T14:51:22,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:22,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:22,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:22,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:22,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:22,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:22,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8942cb50b97d4495902a479ffb78e0c2, entries=150, sequenceid=48, filesize=11.7 K 2024-11-16T14:51:22,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for c410da0e7ea4faea21a8b41f311130b9 in 1088ms, sequenceid=48, compaction requested=true 2024-11-16T14:51:22,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:22,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:22,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:22,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:22,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:22,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:22,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:22,206 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:22,207 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:22,212 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:22,212 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:22,214 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/A is initiating minor compaction (all files) 2024-11-16T14:51:22,214 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/B is initiating minor compaction (all files) 2024-11-16T14:51:22,214 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/A in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:22,214 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/B in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:22,215 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/9f4a21c6272c4d20b7fa084b13348fd0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b34512986b794b4fb352fc865ed073be, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/138192596162420684f9840c18764218] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=44.3 K 2024-11-16T14:51:22,215 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/ff426543b83b4c29bc5d8bbcf2779bde, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/8509cadbced54269bc11ea4eff6aaf24, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/c526034fa3dc4b90b534346dae7af218] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=35.2 K 2024-11-16T14:51:22,217 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ff426543b83b4c29bc5d8bbcf2779bde, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731768679931 2024-11-16T14:51:22,218 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f4a21c6272c4d20b7fa084b13348fd0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731768679931 2024-11-16T14:51:22,218 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8509cadbced54269bc11ea4eff6aaf24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731768680096 2024-11-16T14:51:22,219 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b34512986b794b4fb352fc865ed073be, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731768680091 2024-11-16T14:51:22,220 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 138192596162420684f9840c18764218, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731768680858 2024-11-16T14:51:22,222 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting c526034fa3dc4b90b534346dae7af218, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731768680858 2024-11-16T14:51:22,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T14:51:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:22,327 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-16T14:51:22,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:22,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:22,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:22,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:22,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:22,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:22,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:22,360 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#A#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:22,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/bbf5bd14bc64402e83b861a0039836c0 is 50, key is test_row_0/A:col10/1731768681336/Put/seqid=0 2024-11-16T14:51:22,360 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#B#compaction#9 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:22,361 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/9c2746483a55448098070a7727b19bf5 is 50, key is test_row_0/B:col10/1731768681084/Put/seqid=0 2024-11-16T14:51:22,363 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/8f2f1b0edf544ed7908fc717d45d8137 is 50, key is test_row_0/A:col10/1731768681084/Put/seqid=0 2024-11-16T14:51:22,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741846_1022 (size=12104) 2024-11-16T14:51:22,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741848_1024 (size=12104) 2024-11-16T14:51:22,403 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/9c2746483a55448098070a7727b19bf5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9c2746483a55448098070a7727b19bf5 2024-11-16T14:51:22,407 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/8f2f1b0edf544ed7908fc717d45d8137 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8f2f1b0edf544ed7908fc717d45d8137 2024-11-16T14:51:22,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741847_1023 (size=14341) 2024-11-16T14:51:22,439 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/A of c410da0e7ea4faea21a8b41f311130b9 into 8f2f1b0edf544ed7908fc717d45d8137(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:22,439 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:22,440 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/B of c410da0e7ea4faea21a8b41f311130b9 into 9c2746483a55448098070a7727b19bf5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:22,440 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:22,442 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/A, priority=13, startTime=1731768682187; duration=0sec 2024-11-16T14:51:22,442 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/B, priority=13, startTime=1731768682203; duration=0sec 2024-11-16T14:51:22,443 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:22,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:A 2024-11-16T14:51:22,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:22,445 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:22,445 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:B 2024-11-16T14:51:22,447 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:22,447 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/C is initiating minor compaction (all files) 2024-11-16T14:51:22,447 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/C in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:22,447 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f4e29bfb3b21416d9099ffac8de0d8ed, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4ce8fe50bb7e4a88a45c9a44e9d3a311, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8942cb50b97d4495902a479ffb78e0c2] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=35.2 K 2024-11-16T14:51:22,449 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4e29bfb3b21416d9099ffac8de0d8ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731768679931 2024-11-16T14:51:22,450 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ce8fe50bb7e4a88a45c9a44e9d3a311, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731768680096 2024-11-16T14:51:22,451 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8942cb50b97d4495902a479ffb78e0c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731768680858 2024-11-16T14:51:22,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768742457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:22,493 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:22,494 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/5b49a6a9f54440a68617a6a2cf69d3bd is 50, key is test_row_0/C:col10/1731768681084/Put/seqid=0 2024-11-16T14:51:22,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741849_1025 (size=12104) 2024-11-16T14:51:22,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768742501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:22,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768742512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768742522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:22,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768742530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:22,559 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/5b49a6a9f54440a68617a6a2cf69d3bd as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/5b49a6a9f54440a68617a6a2cf69d3bd 2024-11-16T14:51:22,578 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/C of c410da0e7ea4faea21a8b41f311130b9 into 5b49a6a9f54440a68617a6a2cf69d3bd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:22,578 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:22,578 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/C, priority=13, startTime=1731768682204; duration=0sec 2024-11-16T14:51:22,578 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:22,579 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:C 2024-11-16T14:51:22,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768742581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:22,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:22,650 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:51:22,738 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T14:51:22,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768742802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:22,825 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/bbf5bd14bc64402e83b861a0039836c0 2024-11-16T14:51:22,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/62cf354da1ec45cdb9f212895f2fe8ba is 50, key is test_row_0/B:col10/1731768681336/Put/seqid=0 2024-11-16T14:51:22,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741850_1026 (size=12001) 2024-11-16T14:51:22,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/62cf354da1ec45cdb9f212895f2fe8ba 2024-11-16T14:51:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/1a589f9edb004af79be2a042200a841e is 50, key is test_row_0/C:col10/1731768681336/Put/seqid=0 2024-11-16T14:51:22,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741851_1027 (size=12001) 2024-11-16T14:51:22,936 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/1a589f9edb004af79be2a042200a841e 2024-11-16T14:51:22,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/bbf5bd14bc64402e83b861a0039836c0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/bbf5bd14bc64402e83b861a0039836c0 2024-11-16T14:51:22,963 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T14:51:22,963 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T14:51:22,973 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T14:51:22,973 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T14:51:22,974 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T14:51:22,974 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T14:51:22,976 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T14:51:22,976 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T14:51:22,985 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/bbf5bd14bc64402e83b861a0039836c0, entries=200, sequenceid=84, filesize=14.0 K 2024-11-16T14:51:22,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/62cf354da1ec45cdb9f212895f2fe8ba as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/62cf354da1ec45cdb9f212895f2fe8ba 2024-11-16T14:51:23,007 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/62cf354da1ec45cdb9f212895f2fe8ba, entries=150, sequenceid=84, filesize=11.7 K 2024-11-16T14:51:23,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/1a589f9edb004af79be2a042200a841e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/1a589f9edb004af79be2a042200a841e 2024-11-16T14:51:23,034 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/1a589f9edb004af79be2a042200a841e, entries=150, sequenceid=84, filesize=11.7 K 2024-11-16T14:51:23,039 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for c410da0e7ea4faea21a8b41f311130b9 in 712ms, sequenceid=84, compaction requested=false 2024-11-16T14:51:23,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:23,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:23,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-16T14:51:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-16T14:51:23,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T14:51:23,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1520 sec 2024-11-16T14:51:23,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.1920 sec 2024-11-16T14:51:23,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:51:23,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:23,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:23,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:23,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:23,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:23,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:23,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/056b0cc2618448db854af70f81a83d49 is 50, key is test_row_0/A:col10/1731768683112/Put/seqid=0 2024-11-16T14:51:23,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741852_1028 (size=12001) 2024-11-16T14:51:23,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/056b0cc2618448db854af70f81a83d49 2024-11-16T14:51:23,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/bfb602060920478abba7c879354d07e7 is 50, key is test_row_0/B:col10/1731768683112/Put/seqid=0 2024-11-16T14:51:23,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741853_1029 (size=12001) 2024-11-16T14:51:23,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/bfb602060920478abba7c879354d07e7 2024-11-16T14:51:23,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/83de515188204948ab443e92ff91cee5 is 50, key is test_row_0/C:col10/1731768683112/Put/seqid=0 2024-11-16T14:51:23,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741854_1030 (size=12001) 2024-11-16T14:51:23,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/83de515188204948ab443e92ff91cee5 2024-11-16T14:51:23,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/056b0cc2618448db854af70f81a83d49 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/056b0cc2618448db854af70f81a83d49 2024-11-16T14:51:23,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768683287, max timestamp=1731768683333Num uniques 859; 2024-11-16T14:51:23,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:51:23,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768683287, max timestamp=1731768683333Num uniques 859; 2024-11-16T14:51:23,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:51:23,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/056b0cc2618448db854af70f81a83d49, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T14:51:23,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/bfb602060920478abba7c879354d07e7 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/bfb602060920478abba7c879354d07e7 2024-11-16T14:51:23,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/bfb602060920478abba7c879354d07e7, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T14:51:23,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768683287, max timestamp=1731768683333Num uniques 859; 2024-11-16T14:51:23,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:51:23,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/83de515188204948ab443e92ff91cee5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/83de515188204948ab443e92ff91cee5 2024-11-16T14:51:23,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/83de515188204948ab443e92ff91cee5, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T14:51:23,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=60.38 KB/61830 for c410da0e7ea4faea21a8b41f311130b9 in 296ms, sequenceid=98, compaction requested=true 2024-11-16T14:51:23,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:23,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:23,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:23,412 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:23,412 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:23,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:23,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:23,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:23,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:23,415 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:23,415 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:23,415 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/A is initiating minor compaction (all files) 2024-11-16T14:51:23,415 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/B is initiating minor compaction (all files) 2024-11-16T14:51:23,415 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/A in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:23,415 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/B in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:23,415 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8f2f1b0edf544ed7908fc717d45d8137, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/bbf5bd14bc64402e83b861a0039836c0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/056b0cc2618448db854af70f81a83d49] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=37.5 K 2024-11-16T14:51:23,415 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9c2746483a55448098070a7727b19bf5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/62cf354da1ec45cdb9f212895f2fe8ba, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/bfb602060920478abba7c879354d07e7] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=35.3 K 2024-11-16T14:51:23,417 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f2f1b0edf544ed7908fc717d45d8137, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731768680858 2024-11-16T14:51:23,417 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2746483a55448098070a7727b19bf5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731768680858 2024-11-16T14:51:23,418 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf5bd14bc64402e83b861a0039836c0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731768681336 2024-11-16T14:51:23,419 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 056b0cc2618448db854af70f81a83d49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731768682435 2024-11-16T14:51:23,419 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 62cf354da1ec45cdb9f212895f2fe8ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731768681336 2024-11-16T14:51:23,420 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb602060920478abba7c879354d07e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731768682435 2024-11-16T14:51:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:23,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=67.09 KB heapSize=185.77 KB 2024-11-16T14:51:23,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:23,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:23,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:23,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:23,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:23,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:23,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/6e1a413203b74aa9ae6d68e3b7f0f98f is 50, key is test_row_0/A:col10/1731768683316/Put/seqid=0 2024-11-16T14:51:23,454 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:23,455 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/db390dfacd6b4379bfbaf34ddcf55c46 is 50, key is test_row_0/B:col10/1731768683112/Put/seqid=0 2024-11-16T14:51:23,458 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#A#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:23,459 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/70b69d3d6b8344779e598a6001330a05 is 50, key is test_row_0/A:col10/1731768683112/Put/seqid=0 2024-11-16T14:51:23,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741855_1031 (size=14341) 2024-11-16T14:51:23,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/6e1a413203b74aa9ae6d68e3b7f0f98f 2024-11-16T14:51:23,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741856_1032 (size=12207) 2024-11-16T14:51:23,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741857_1033 (size=12207) 2024-11-16T14:51:23,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/346119fc0e5040aebc953d963c1beafe is 50, key is test_row_0/B:col10/1731768683316/Put/seqid=0 2024-11-16T14:51:23,528 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/70b69d3d6b8344779e598a6001330a05 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/70b69d3d6b8344779e598a6001330a05 2024-11-16T14:51:23,531 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/db390dfacd6b4379bfbaf34ddcf55c46 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/db390dfacd6b4379bfbaf34ddcf55c46 2024-11-16T14:51:23,547 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/A of c410da0e7ea4faea21a8b41f311130b9 into 70b69d3d6b8344779e598a6001330a05(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:23,548 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:23,548 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/A, priority=13, startTime=1731768683412; duration=0sec 2024-11-16T14:51:23,548 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:23,549 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:A 2024-11-16T14:51:23,549 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:23,551 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/B of c410da0e7ea4faea21a8b41f311130b9 into db390dfacd6b4379bfbaf34ddcf55c46(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:23,551 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:23,551 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/B, priority=13, startTime=1731768683412; duration=0sec 2024-11-16T14:51:23,551 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:23,552 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:B 2024-11-16T14:51:23,552 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:23,552 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/C is initiating minor compaction (all files) 2024-11-16T14:51:23,552 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/C in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:23,552 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/5b49a6a9f54440a68617a6a2cf69d3bd, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/1a589f9edb004af79be2a042200a841e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/83de515188204948ab443e92ff91cee5] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=35.3 K 2024-11-16T14:51:23,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741858_1034 (size=12001) 2024-11-16T14:51:23,554 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b49a6a9f54440a68617a6a2cf69d3bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731768680858 2024-11-16T14:51:23,554 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a589f9edb004af79be2a042200a841e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731768681336 2024-11-16T14:51:23,555 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83de515188204948ab443e92ff91cee5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731768682435 2024-11-16T14:51:23,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768743549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768743551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768743551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768743553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768743553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,571 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in c410da0e7ea4faea21a8b41f311130b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,577 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:23,578 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/5ec535cfcd254c61bb279ccc95b984f8 is 50, key is test_row_0/C:col10/1731768683112/Put/seqid=0 2024-11-16T14:51:23,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741859_1035 (size=12207) 2024-11-16T14:51:23,613 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/5ec535cfcd254c61bb279ccc95b984f8 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/5ec535cfcd254c61bb279ccc95b984f8 2024-11-16T14:51:23,627 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/C of c410da0e7ea4faea21a8b41f311130b9 into 5ec535cfcd254c61bb279ccc95b984f8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:23,627 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:23,627 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/C, priority=13, startTime=1731768683413; duration=0sec 2024-11-16T14:51:23,628 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:23,628 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:C 2024-11-16T14:51:23,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768743671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768743672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768743674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768743674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768743882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768743882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768743882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768743891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:23,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:23,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/346119fc0e5040aebc953d963c1beafe 2024-11-16T14:51:23,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/ada52e2fa0024730a79a8cc84eb68fed is 50, key is test_row_0/C:col10/1731768683316/Put/seqid=0 2024-11-16T14:51:23,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741860_1036 (size=12001) 2024-11-16T14:51:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T14:51:24,040 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:24,041 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-16T14:51:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:51:24,045 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:24,046 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:24,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:51:24,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768744191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768744191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:51:24,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:24,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:24,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:24,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:24,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768744201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768744201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:24,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:51:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:24,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:24,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:51:24,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:24,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:24,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/ada52e2fa0024730a79a8cc84eb68fed 2024-11-16T14:51:24,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/6e1a413203b74aa9ae6d68e3b7f0f98f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/6e1a413203b74aa9ae6d68e3b7f0f98f 2024-11-16T14:51:24,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/6e1a413203b74aa9ae6d68e3b7f0f98f, entries=200, sequenceid=125, filesize=14.0 K 2024-11-16T14:51:24,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/346119fc0e5040aebc953d963c1beafe as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/346119fc0e5040aebc953d963c1beafe 2024-11-16T14:51:24,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/346119fc0e5040aebc953d963c1beafe, entries=150, sequenceid=125, filesize=11.7 K 2024-11-16T14:51:24,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/ada52e2fa0024730a79a8cc84eb68fed as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/ada52e2fa0024730a79a8cc84eb68fed 2024-11-16T14:51:24,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/ada52e2fa0024730a79a8cc84eb68fed, entries=150, sequenceid=125, filesize=11.7 K 2024-11-16T14:51:24,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.72 KB/190176, currentSize=114.05 KB/116790 for c410da0e7ea4faea21a8b41f311130b9 in 1010ms, sequenceid=125, compaction requested=false 2024-11-16T14:51:24,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:24,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T14:51:24,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:24,516 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:51:24,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:24,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:24,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:24,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:24,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:24,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:24,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/09861853e87f4d1b972419eb90ecceb9 is 50, key is test_row_0/A:col10/1731768683550/Put/seqid=0 2024-11-16T14:51:24,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741861_1037 (size=12151) 2024-11-16T14:51:24,588 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/09861853e87f4d1b972419eb90ecceb9 2024-11-16T14:51:24,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/15561e60737f484c80177cd36de85c65 is 50, key is test_row_0/B:col10/1731768683550/Put/seqid=0 2024-11-16T14:51:24,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741862_1038 (size=12151) 2024-11-16T14:51:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:51:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:24,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:24,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768744716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768744718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768744721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768744726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768744830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768744831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768744831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:24,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768744841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768745040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768745041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768745041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768745050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,053 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/15561e60737f484c80177cd36de85c65 2024-11-16T14:51:25,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/8611ccbf2ca7465284a788e768c13b9e is 50, key is test_row_0/C:col10/1731768683550/Put/seqid=0 2024-11-16T14:51:25,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741863_1039 (size=12151) 2024-11-16T14:51:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:51:25,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768745351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768745351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768745351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768745360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,496 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/8611ccbf2ca7465284a788e768c13b9e 2024-11-16T14:51:25,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/09861853e87f4d1b972419eb90ecceb9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/09861853e87f4d1b972419eb90ecceb9 2024-11-16T14:51:25,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/09861853e87f4d1b972419eb90ecceb9, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T14:51:25,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/15561e60737f484c80177cd36de85c65 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/15561e60737f484c80177cd36de85c65 2024-11-16T14:51:25,534 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/15561e60737f484c80177cd36de85c65, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T14:51:25,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/8611ccbf2ca7465284a788e768c13b9e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8611ccbf2ca7465284a788e768c13b9e 2024-11-16T14:51:25,551 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8611ccbf2ca7465284a788e768c13b9e, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T14:51:25,553 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c410da0e7ea4faea21a8b41f311130b9 in 1038ms, sequenceid=148, compaction requested=true 2024-11-16T14:51:25,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:25,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:25,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-16T14:51:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-16T14:51:25,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.5240 sec 2024-11-16T14:51:25,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-16T14:51:25,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5120 sec 2024-11-16T14:51:25,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:51:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:25,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:25,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/f18ef46f94cf40abab026d4ff4088a3c is 50, key is test_row_0/A:col10/1731768685582/Put/seqid=0 2024-11-16T14:51:25,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741864_1040 (size=14541) 2024-11-16T14:51:25,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/f18ef46f94cf40abab026d4ff4088a3c 2024-11-16T14:51:25,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/4537cd17c1c242638b6e409a23b63094 is 50, key is test_row_0/B:col10/1731768685582/Put/seqid=0 2024-11-16T14:51:25,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741865_1041 (size=12151) 2024-11-16T14:51:25,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/4537cd17c1c242638b6e409a23b63094 2024-11-16T14:51:25,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/6a59f9a55e0b4bd3b0e8ad947e82a1f0 is 50, key is test_row_0/C:col10/1731768685582/Put/seqid=0 2024-11-16T14:51:25,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741866_1042 (size=12151) 2024-11-16T14:51:25,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/6a59f9a55e0b4bd3b0e8ad947e82a1f0 2024-11-16T14:51:25,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768745728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/f18ef46f94cf40abab026d4ff4088a3c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f18ef46f94cf40abab026d4ff4088a3c 2024-11-16T14:51:25,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f18ef46f94cf40abab026d4ff4088a3c, entries=200, sequenceid=161, filesize=14.2 K 2024-11-16T14:51:25,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/4537cd17c1c242638b6e409a23b63094 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4537cd17c1c242638b6e409a23b63094 2024-11-16T14:51:25,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4537cd17c1c242638b6e409a23b63094, entries=150, sequenceid=161, filesize=11.9 K 2024-11-16T14:51:25,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/6a59f9a55e0b4bd3b0e8ad947e82a1f0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/6a59f9a55e0b4bd3b0e8ad947e82a1f0 2024-11-16T14:51:25,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/6a59f9a55e0b4bd3b0e8ad947e82a1f0, entries=150, sequenceid=161, filesize=11.9 K 2024-11-16T14:51:25,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c410da0e7ea4faea21a8b41f311130b9 in 191ms, sequenceid=161, compaction requested=true 2024-11-16T14:51:25,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:25,775 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:25,776 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:25,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:25,779 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53240 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:25,779 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:25,779 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/A is initiating minor compaction (all files) 2024-11-16T14:51:25,779 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/B is initiating minor compaction (all files) 2024-11-16T14:51:25,779 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/B in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:25,779 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/A in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:25,779 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/70b69d3d6b8344779e598a6001330a05, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/6e1a413203b74aa9ae6d68e3b7f0f98f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/09861853e87f4d1b972419eb90ecceb9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f18ef46f94cf40abab026d4ff4088a3c] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=52.0 K 2024-11-16T14:51:25,779 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/db390dfacd6b4379bfbaf34ddcf55c46, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/346119fc0e5040aebc953d963c1beafe, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/15561e60737f484c80177cd36de85c65, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4537cd17c1c242638b6e409a23b63094] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=47.4 K 2024-11-16T14:51:25,780 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70b69d3d6b8344779e598a6001330a05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731768682435 2024-11-16T14:51:25,781 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting db390dfacd6b4379bfbaf34ddcf55c46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731768682435 2024-11-16T14:51:25,781 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 346119fc0e5040aebc953d963c1beafe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731768683316 2024-11-16T14:51:25,781 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1a413203b74aa9ae6d68e3b7f0f98f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731768683316 2024-11-16T14:51:25,782 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 15561e60737f484c80177cd36de85c65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731768683525 2024-11-16T14:51:25,782 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09861853e87f4d1b972419eb90ecceb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731768683525 2024-11-16T14:51:25,783 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f18ef46f94cf40abab026d4ff4088a3c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731768684714 2024-11-16T14:51:25,783 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4537cd17c1c242638b6e409a23b63094, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731768684716 2024-11-16T14:51:25,802 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#B#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:25,802 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/fc2b31a910964548802a4b56c97cd7f4 is 50, key is test_row_0/B:col10/1731768685582/Put/seqid=0 2024-11-16T14:51:25,814 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#A#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:25,815 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/8ba3284ef1204c32986fee9e9cc17596 is 50, key is test_row_0/A:col10/1731768685582/Put/seqid=0 2024-11-16T14:51:25,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741867_1043 (size=12493) 2024-11-16T14:51:25,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741868_1044 (size=12493) 2024-11-16T14:51:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:25,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:51:25,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:25,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:25,847 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/fc2b31a910964548802a4b56c97cd7f4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fc2b31a910964548802a4b56c97cd7f4 2024-11-16T14:51:25,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:25,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:25,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:25,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:25,852 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/8ba3284ef1204c32986fee9e9cc17596 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8ba3284ef1204c32986fee9e9cc17596 2024-11-16T14:51:25,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/ed81d72f05c34747a0773f895b42ff92 is 50, key is test_row_0/A:col10/1731768685845/Put/seqid=0 2024-11-16T14:51:25,867 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/A of c410da0e7ea4faea21a8b41f311130b9 into 8ba3284ef1204c32986fee9e9cc17596(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:25,867 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:25,867 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/A, priority=12, startTime=1731768685775; duration=0sec 2024-11-16T14:51:25,868 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:25,868 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:A 2024-11-16T14:51:25,868 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:25,868 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/B of c410da0e7ea4faea21a8b41f311130b9 into fc2b31a910964548802a4b56c97cd7f4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:25,869 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:25,869 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/B, priority=12, startTime=1731768685776; duration=0sec 2024-11-16T14:51:25,872 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:25,872 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/C is initiating minor compaction (all files) 2024-11-16T14:51:25,872 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/C in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:25,872 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/5ec535cfcd254c61bb279ccc95b984f8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/ada52e2fa0024730a79a8cc84eb68fed, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8611ccbf2ca7465284a788e768c13b9e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/6a59f9a55e0b4bd3b0e8ad947e82a1f0] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=47.4 K 2024-11-16T14:51:25,872 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:25,872 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:B 2024-11-16T14:51:25,873 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ec535cfcd254c61bb279ccc95b984f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731768682435 2024-11-16T14:51:25,874 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ada52e2fa0024730a79a8cc84eb68fed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731768683316 2024-11-16T14:51:25,875 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8611ccbf2ca7465284a788e768c13b9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731768683525 2024-11-16T14:51:25,876 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a59f9a55e0b4bd3b0e8ad947e82a1f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731768684716 2024-11-16T14:51:25,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741869_1045 (size=14541) 2024-11-16T14:51:25,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/ed81d72f05c34747a0773f895b42ff92 2024-11-16T14:51:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768745875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768745884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768745887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768745891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768745885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:25,909 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:25,910 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/9da6659ac17548e59e8a1e69df5181a5 is 50, key is test_row_0/C:col10/1731768685582/Put/seqid=0 2024-11-16T14:51:25,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/fba5013667d54b45a581e8a758959e97 is 50, key is test_row_0/B:col10/1731768685845/Put/seqid=0 2024-11-16T14:51:25,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741870_1046 (size=12493) 2024-11-16T14:51:25,969 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/9da6659ac17548e59e8a1e69df5181a5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/9da6659ac17548e59e8a1e69df5181a5 2024-11-16T14:51:25,979 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/C of c410da0e7ea4faea21a8b41f311130b9 into 9da6659ac17548e59e8a1e69df5181a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:25,979 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:25,980 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/C, priority=12, startTime=1731768685776; duration=0sec 2024-11-16T14:51:25,980 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:25,980 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:C 2024-11-16T14:51:25,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741871_1047 (size=12151) 2024-11-16T14:51:26,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768746001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768746011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768746011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768746011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768746011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T14:51:26,191 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:26,192 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-16T14:51:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:51:26,200 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:26,202 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:26,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768746211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768746221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768746221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768746224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768746224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:51:26,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T14:51:26,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:26,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:26,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:26,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/fba5013667d54b45a581e8a758959e97 2024-11-16T14:51:26,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/e336bea89474483981f8c9e1e69ad7b9 is 50, key is test_row_0/C:col10/1731768685845/Put/seqid=0 2024-11-16T14:51:26,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741872_1048 (size=12151) 2024-11-16T14:51:26,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T14:51:26,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:26,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:26,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:26,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:51:26,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768746521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768746531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768746531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768746531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768746531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:26,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T14:51:26,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:26,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:26,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:26,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/e336bea89474483981f8c9e1e69ad7b9 2024-11-16T14:51:26,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T14:51:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:26,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:51:26,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/ed81d72f05c34747a0773f895b42ff92 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/ed81d72f05c34747a0773f895b42ff92 2024-11-16T14:51:26,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/ed81d72f05c34747a0773f895b42ff92, entries=200, sequenceid=181, filesize=14.2 K 2024-11-16T14:51:26,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/fba5013667d54b45a581e8a758959e97 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fba5013667d54b45a581e8a758959e97 2024-11-16T14:51:26,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fba5013667d54b45a581e8a758959e97, entries=150, sequenceid=181, filesize=11.9 K 2024-11-16T14:51:26,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/e336bea89474483981f8c9e1e69ad7b9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/e336bea89474483981f8c9e1e69ad7b9 2024-11-16T14:51:26,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/e336bea89474483981f8c9e1e69ad7b9, entries=150, sequenceid=181, filesize=11.9 K 2024-11-16T14:51:26,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c410da0e7ea4faea21a8b41f311130b9 in 1028ms, sequenceid=181, compaction requested=false 2024-11-16T14:51:26,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:26,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T14:51:26,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:26,986 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:51:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:26,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:26,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/630bb7a11abf4b058d8255bbf8741ead is 50, key is test_row_0/A:col10/1731768685881/Put/seqid=0 2024-11-16T14:51:27,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741873_1049 (size=12151) 2024-11-16T14:51:27,030 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/630bb7a11abf4b058d8255bbf8741ead 2024-11-16T14:51:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:27,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:27,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/747aac5f0b2c418fb7fadf11d6d58bf7 is 50, key is test_row_0/B:col10/1731768685881/Put/seqid=0 2024-11-16T14:51:27,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741874_1050 (size=12151) 2024-11-16T14:51:27,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768747081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731768687072, max timestamp=1731768687089Num uniques 859; 2024-11-16T14:51:27,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T14:51:27,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731768687072, max timestamp=1731768687089Num uniques 859; 2024-11-16T14:51:27,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T14:51:27,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731768687085, max timestamp=1731768687089Num uniques 859; 2024-11-16T14:51:27,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-16T14:51:27,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768747130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768747132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768747132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768747137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768747201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768747251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768747251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768747251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768747251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:51:27,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768747411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/747aac5f0b2c418fb7fadf11d6d58bf7 2024-11-16T14:51:27,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768747460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768747461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768747461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768747461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/681f1ab4559546ec8689172a9d9256f1 is 50, key is test_row_0/C:col10/1731768685881/Put/seqid=0 2024-11-16T14:51:27,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741875_1051 (size=12151) 2024-11-16T14:51:27,489 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/681f1ab4559546ec8689172a9d9256f1 2024-11-16T14:51:27,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/630bb7a11abf4b058d8255bbf8741ead as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/630bb7a11abf4b058d8255bbf8741ead 2024-11-16T14:51:27,508 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/630bb7a11abf4b058d8255bbf8741ead, entries=150, sequenceid=196, filesize=11.9 K 2024-11-16T14:51:27,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/747aac5f0b2c418fb7fadf11d6d58bf7 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/747aac5f0b2c418fb7fadf11d6d58bf7 2024-11-16T14:51:27,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/747aac5f0b2c418fb7fadf11d6d58bf7, entries=150, sequenceid=196, filesize=11.9 K 2024-11-16T14:51:27,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/681f1ab4559546ec8689172a9d9256f1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/681f1ab4559546ec8689172a9d9256f1 2024-11-16T14:51:27,533 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/681f1ab4559546ec8689172a9d9256f1, entries=150, sequenceid=196, filesize=11.9 K 2024-11-16T14:51:27,542 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.37 KB/123255 for c410da0e7ea4faea21a8b41f311130b9 in 557ms, sequenceid=196, compaction requested=true 2024-11-16T14:51:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-16T14:51:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-16T14:51:27,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-16T14:51:27,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3420 sec 2024-11-16T14:51:27,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3570 sec 2024-11-16T14:51:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:27,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=127.08 KB heapSize=354.77 KB 2024-11-16T14:51:27,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:27,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:27,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:27,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:27,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:27,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:27,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/b72ddefc95e4477896e0b69a0a679129 is 50, key is test_row_0/A:col10/1731768687132/Put/seqid=0 2024-11-16T14:51:27,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741876_1052 (size=14541) 2024-11-16T14:51:27,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768747777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768747782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768747783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768747783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768747784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768747890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768747901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768747901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768747901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768747901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768748101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768748121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768748121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768748121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768748122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/b72ddefc95e4477896e0b69a0a679129 2024-11-16T14:51:28,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/715cf51311b04a1dbf736095ddefb1df is 50, key is test_row_0/B:col10/1731768687132/Put/seqid=0 2024-11-16T14:51:28,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741877_1053 (size=12151) 2024-11-16T14:51:28,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/715cf51311b04a1dbf736095ddefb1df 2024-11-16T14:51:28,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/a5c78557fd4e41bd8ed66e08e5d3f6a6 is 50, key is test_row_0/C:col10/1731768687132/Put/seqid=0 2024-11-16T14:51:28,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741878_1054 (size=12151) 2024-11-16T14:51:28,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/a5c78557fd4e41bd8ed66e08e5d3f6a6 2024-11-16T14:51:28,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/b72ddefc95e4477896e0b69a0a679129 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b72ddefc95e4477896e0b69a0a679129 2024-11-16T14:51:28,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b72ddefc95e4477896e0b69a0a679129, entries=200, sequenceid=230, filesize=14.2 K 2024-11-16T14:51:28,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/715cf51311b04a1dbf736095ddefb1df as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/715cf51311b04a1dbf736095ddefb1df 2024-11-16T14:51:28,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/715cf51311b04a1dbf736095ddefb1df, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T14:51:28,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/a5c78557fd4e41bd8ed66e08e5d3f6a6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/a5c78557fd4e41bd8ed66e08e5d3f6a6 2024-11-16T14:51:28,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/a5c78557fd4e41bd8ed66e08e5d3f6a6, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T14:51:28,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~354.73 KB/363240, currentSize=67.09 KB/68700 for c410da0e7ea4faea21a8b41f311130b9 in 559ms, sequenceid=230, compaction requested=true 2024-11-16T14:51:28,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:28,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:28,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:28,284 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:28,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:28,284 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:28,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:28,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:28,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:28,286 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:28,286 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:28,286 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/A is initiating minor compaction (all files) 2024-11-16T14:51:28,286 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/B is initiating minor compaction (all files) 2024-11-16T14:51:28,286 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/A in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,286 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/B in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,286 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fc2b31a910964548802a4b56c97cd7f4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fba5013667d54b45a581e8a758959e97, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/747aac5f0b2c418fb7fadf11d6d58bf7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/715cf51311b04a1dbf736095ddefb1df] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=47.8 K 2024-11-16T14:51:28,286 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8ba3284ef1204c32986fee9e9cc17596, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/ed81d72f05c34747a0773f895b42ff92, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/630bb7a11abf4b058d8255bbf8741ead, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b72ddefc95e4477896e0b69a0a679129] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=52.5 K 2024-11-16T14:51:28,287 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting fc2b31a910964548802a4b56c97cd7f4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731768684716 2024-11-16T14:51:28,287 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ba3284ef1204c32986fee9e9cc17596, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731768684716 2024-11-16T14:51:28,287 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed81d72f05c34747a0773f895b42ff92, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731768685681 2024-11-16T14:51:28,287 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting fba5013667d54b45a581e8a758959e97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731768685681 2024-11-16T14:51:28,287 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 630bb7a11abf4b058d8255bbf8741ead, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731768685881 2024-11-16T14:51:28,287 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 747aac5f0b2c418fb7fadf11d6d58bf7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731768685881 2024-11-16T14:51:28,288 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b72ddefc95e4477896e0b69a0a679129, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768687098 2024-11-16T14:51:28,288 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 715cf51311b04a1dbf736095ddefb1df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768687128 2024-11-16T14:51:28,309 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#A#compaction#42 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:28,310 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/85da1d14c2524f378bf1ce296c882c31 is 50, key is test_row_0/A:col10/1731768687132/Put/seqid=0 2024-11-16T14:51:28,316 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#B#compaction#43 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:28,317 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/953178a15aa0474fa8bc25f3165dd59b is 50, key is test_row_0/B:col10/1731768687132/Put/seqid=0 2024-11-16T14:51:28,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741879_1055 (size=12629) 2024-11-16T14:51:28,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741880_1056 (size=12629) 2024-11-16T14:51:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T14:51:28,351 INFO [Async-Client-Retry-Timer-pool-0 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:28,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-16T14:51:28,356 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:51:28,357 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:28,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:28,359 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/85da1d14c2524f378bf1ce296c882c31 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/85da1d14c2524f378bf1ce296c882c31 2024-11-16T14:51:28,374 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/A of c410da0e7ea4faea21a8b41f311130b9 into 85da1d14c2524f378bf1ce296c882c31(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:28,374 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:28,375 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/A, priority=12, startTime=1731768688284; duration=0sec 2024-11-16T14:51:28,375 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:28,375 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:A 2024-11-16T14:51:28,375 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:28,377 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:28,377 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/C is initiating minor compaction (all files) 2024-11-16T14:51:28,378 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/C in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,378 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/9da6659ac17548e59e8a1e69df5181a5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/e336bea89474483981f8c9e1e69ad7b9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/681f1ab4559546ec8689172a9d9256f1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/a5c78557fd4e41bd8ed66e08e5d3f6a6] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=47.8 K 2024-11-16T14:51:28,379 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9da6659ac17548e59e8a1e69df5181a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731768684716 2024-11-16T14:51:28,380 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting e336bea89474483981f8c9e1e69ad7b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731768685681 2024-11-16T14:51:28,381 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 681f1ab4559546ec8689172a9d9256f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731768685881 2024-11-16T14:51:28,381 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c78557fd4e41bd8ed66e08e5d3f6a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768687128 2024-11-16T14:51:28,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:51:28,418 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#C#compaction#44 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:28,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:28,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:28,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:28,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:28,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:28,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:28,425 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/f242d70ae8554736bbd9828919928f9c is 50, key is test_row_0/C:col10/1731768687132/Put/seqid=0 2024-11-16T14:51:28,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/4dc5a9579a44487f87dc672e73407ddc is 50, key is test_row_0/A:col10/1731768687765/Put/seqid=0 2024-11-16T14:51:28,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741881_1057 (size=12629) 2024-11-16T14:51:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:51:28,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741882_1058 (size=12151) 2024-11-16T14:51:28,476 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/f242d70ae8554736bbd9828919928f9c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f242d70ae8554736bbd9828919928f9c 2024-11-16T14:51:28,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768748468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768748469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768748472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768748474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768748469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,487 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/C of c410da0e7ea4faea21a8b41f311130b9 into f242d70ae8554736bbd9828919928f9c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:28,487 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:28,487 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/C, priority=12, startTime=1731768688284; duration=0sec 2024-11-16T14:51:28,487 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:28,487 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:C 2024-11-16T14:51:28,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:51:28,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:28,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768748590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768748590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768748591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768748591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768748591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:51:28,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:28,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:51:28,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,756 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/953178a15aa0474fa8bc25f3165dd59b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/953178a15aa0474fa8bc25f3165dd59b 2024-11-16T14:51:28,763 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/B of c410da0e7ea4faea21a8b41f311130b9 into 953178a15aa0474fa8bc25f3165dd59b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:28,764 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:28,764 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/B, priority=12, startTime=1731768688284; duration=0sec 2024-11-16T14:51:28,764 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:28,764 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:B 2024-11-16T14:51:28,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768748801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768748801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768748801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768748802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768748802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:28,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:51:28,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:28,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/4dc5a9579a44487f87dc672e73407ddc 2024-11-16T14:51:28,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/7cf0835942bf4b749ab4f3faa43e38f6 is 50, key is test_row_0/B:col10/1731768687765/Put/seqid=0 2024-11-16T14:51:28,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741883_1059 (size=12151) 2024-11-16T14:51:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:51:28,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:51:28,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:28,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:28,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768749112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:29,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768749112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:29,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768749112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768749112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:29,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768749112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:29,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:51:29,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:29,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:51:29,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:29,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/7cf0835942bf4b749ab4f3faa43e38f6 2024-11-16T14:51:29,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d0a4592fb2de49a7b5157a86553608b7 is 50, key is test_row_0/C:col10/1731768687765/Put/seqid=0 2024-11-16T14:51:29,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741884_1060 (size=12151) 2024-11-16T14:51:29,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:51:29,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:29,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:51:29,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:51:29,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:29,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768749620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:29,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:29,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768749620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:29,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:29,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768749630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768749631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768749631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:29,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:29,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:29,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d0a4592fb2de49a7b5157a86553608b7 2024-11-16T14:51:29,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/4dc5a9579a44487f87dc672e73407ddc as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4dc5a9579a44487f87dc672e73407ddc 2024-11-16T14:51:29,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4dc5a9579a44487f87dc672e73407ddc, entries=150, sequenceid=245, filesize=11.9 K 2024-11-16T14:51:29,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:51:29,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/7cf0835942bf4b749ab4f3faa43e38f6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/7cf0835942bf4b749ab4f3faa43e38f6 2024-11-16T14:51:29,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:29,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:29,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/7cf0835942bf4b749ab4f3faa43e38f6, entries=150, sequenceid=245, filesize=11.9 K 2024-11-16T14:51:29,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d0a4592fb2de49a7b5157a86553608b7 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0a4592fb2de49a7b5157a86553608b7 2024-11-16T14:51:29,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0a4592fb2de49a7b5157a86553608b7, entries=150, sequenceid=245, filesize=11.9 K 2024-11-16T14:51:29,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c410da0e7ea4faea21a8b41f311130b9 in 1375ms, sequenceid=245, compaction requested=false 2024-11-16T14:51:29,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:29,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T14:51:29,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:29,928 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T14:51:29,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:29,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:29,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:29,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:29,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:29,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:29,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/f613da36a79e48598ec534b60b215cd9 is 50, key is test_row_0/A:col10/1731768688469/Put/seqid=0 2024-11-16T14:51:29,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741885_1061 (size=12301) 2024-11-16T14:51:29,939 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/f613da36a79e48598ec534b60b215cd9 2024-11-16T14:51:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/b4d6900670974d028ebf741b9b2cbfc1 is 50, key is test_row_0/B:col10/1731768688469/Put/seqid=0 2024-11-16T14:51:29,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741886_1062 (size=12301) 2024-11-16T14:51:29,987 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/b4d6900670974d028ebf741b9b2cbfc1 2024-11-16T14:51:30,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/0a68f9d9fd4f409cb6fd80012aa0775a is 50, key is test_row_0/C:col10/1731768688469/Put/seqid=0 2024-11-16T14:51:30,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741887_1063 (size=12301) 2024-11-16T14:51:30,018 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/0a68f9d9fd4f409cb6fd80012aa0775a 2024-11-16T14:51:30,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/f613da36a79e48598ec534b60b215cd9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f613da36a79e48598ec534b60b215cd9 2024-11-16T14:51:30,038 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f613da36a79e48598ec534b60b215cd9, entries=150, sequenceid=265, filesize=12.0 K 2024-11-16T14:51:30,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/b4d6900670974d028ebf741b9b2cbfc1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/b4d6900670974d028ebf741b9b2cbfc1 2024-11-16T14:51:30,049 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/b4d6900670974d028ebf741b9b2cbfc1, entries=150, sequenceid=265, filesize=12.0 K 2024-11-16T14:51:30,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/0a68f9d9fd4f409cb6fd80012aa0775a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/0a68f9d9fd4f409cb6fd80012aa0775a 2024-11-16T14:51:30,065 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/0a68f9d9fd4f409cb6fd80012aa0775a, entries=150, sequenceid=265, filesize=12.0 K 2024-11-16T14:51:30,067 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for c410da0e7ea4faea21a8b41f311130b9 in 138ms, sequenceid=265, compaction requested=true 2024-11-16T14:51:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-16T14:51:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-16T14:51:30,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T14:51:30,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7130 sec 2024-11-16T14:51:30,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7230 sec 2024-11-16T14:51:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T14:51:30,501 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:30,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:30,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-16T14:51:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:51:30,505 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:30,506 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:30,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:51:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:30,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:51:30,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:30,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:30,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:30,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:30,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:30,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:30,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/4675e01685f548c5a31ff3414dac83a6 is 50, key is test_row_0/A:col10/1731768690644/Put/seqid=0 2024-11-16T14:51:30,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T14:51:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:30,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:30,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:30,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741888_1064 (size=12301) 2024-11-16T14:51:30,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768690675, max timestamp=1731768690689Num uniques 859; 2024-11-16T14:51:30,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:51:30,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768690675, max timestamp=1731768690689Num uniques 859; 2024-11-16T14:51:30,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:51:30,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768690675, max timestamp=1731768690689Num uniques 859; 2024-11-16T14:51:30,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:51:30,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768750721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:30,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768750721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:30,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768750723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:30,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768750724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768750724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:30,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T14:51:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:30,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:51:30,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768750831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768750831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768750831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:30,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768750840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:30,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768750840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:30,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T14:51:30,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:30,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:30,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:30,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:30,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:31,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768751050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768751051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768751051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768751051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768751051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/4675e01685f548c5a31ff3414dac83a6 2024-11-16T14:51:31,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/4cabd4ae79774f2289db85722a11156b is 50, key is test_row_0/B:col10/1731768690644/Put/seqid=0 2024-11-16T14:51:31,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741889_1065 (size=12301) 2024-11-16T14:51:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:51:31,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T14:51:31,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:31,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:31,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:31,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:31,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:31,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T14:51:31,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:31,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:31,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:31,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:31,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:31,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768751361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768751361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768751361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768751361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768751361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T14:51:31,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:31,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:31,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/4cabd4ae79774f2289db85722a11156b 2024-11-16T14:51:31,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/592e33cb967f4a0695eb72311d846fb3 is 50, key is test_row_0/C:col10/1731768690644/Put/seqid=0 2024-11-16T14:51:31,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741890_1066 (size=12301) 2024-11-16T14:51:31,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/592e33cb967f4a0695eb72311d846fb3 2024-11-16T14:51:31,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/4675e01685f548c5a31ff3414dac83a6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4675e01685f548c5a31ff3414dac83a6 2024-11-16T14:51:31,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4675e01685f548c5a31ff3414dac83a6, entries=150, sequenceid=276, filesize=12.0 K 2024-11-16T14:51:31,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/4cabd4ae79774f2289db85722a11156b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4cabd4ae79774f2289db85722a11156b 2024-11-16T14:51:31,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4cabd4ae79774f2289db85722a11156b, entries=150, sequenceid=276, filesize=12.0 K 2024-11-16T14:51:31,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/592e33cb967f4a0695eb72311d846fb3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/592e33cb967f4a0695eb72311d846fb3 2024-11-16T14:51:31,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/592e33cb967f4a0695eb72311d846fb3, entries=150, sequenceid=276, filesize=12.0 K 2024-11-16T14:51:31,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for c410da0e7ea4faea21a8b41f311130b9 in 947ms, sequenceid=276, compaction requested=true 2024-11-16T14:51:31,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:31,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:31,594 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:31,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:31,594 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:31,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:31,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:31,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:31,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:31,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T14:51:31,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:31,596 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T14:51:31,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:31,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:31,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:31,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:31,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:31,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:31,598 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:31,598 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/B is initiating minor compaction (all files) 2024-11-16T14:51:31,598 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/B in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:31,598 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/953178a15aa0474fa8bc25f3165dd59b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/7cf0835942bf4b749ab4f3faa43e38f6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/b4d6900670974d028ebf741b9b2cbfc1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4cabd4ae79774f2289db85722a11156b] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=48.2 K 2024-11-16T14:51:31,598 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:31,598 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/A is initiating minor compaction (all files) 2024-11-16T14:51:31,598 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/A in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:31,599 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/85da1d14c2524f378bf1ce296c882c31, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4dc5a9579a44487f87dc672e73407ddc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f613da36a79e48598ec534b60b215cd9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4675e01685f548c5a31ff3414dac83a6] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=48.2 K 2024-11-16T14:51:31,599 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 953178a15aa0474fa8bc25f3165dd59b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768687128 2024-11-16T14:51:31,599 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85da1d14c2524f378bf1ce296c882c31, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768687128 2024-11-16T14:51:31,599 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cf0835942bf4b749ab4f3faa43e38f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731768687765 2024-11-16T14:51:31,600 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc5a9579a44487f87dc672e73407ddc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731768687765 2024-11-16T14:51:31,600 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting b4d6900670974d028ebf741b9b2cbfc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731768688462 2024-11-16T14:51:31,600 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f613da36a79e48598ec534b60b215cd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731768688462 2024-11-16T14:51:31,601 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4675e01685f548c5a31ff3414dac83a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731768690638 2024-11-16T14:51:31,602 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cabd4ae79774f2289db85722a11156b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731768690638 2024-11-16T14:51:31,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/3ff2b0c436cf43f48df2d9dc099ce670 is 50, key is test_row_0/A:col10/1731768690722/Put/seqid=0 2024-11-16T14:51:31,629 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#B#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:31,629 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#A#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:31,630 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/fe9b621d4c3c4f2c8dec6584c93fd07c is 50, key is test_row_0/B:col10/1731768690644/Put/seqid=0 2024-11-16T14:51:31,630 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/877a9c4bc75c4a099fd320313f1f68c0 is 50, key is test_row_0/A:col10/1731768690644/Put/seqid=0 2024-11-16T14:51:31,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741891_1067 (size=12301) 2024-11-16T14:51:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:51:31,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741892_1068 (size=12915) 2024-11-16T14:51:31,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741893_1069 (size=12915) 2024-11-16T14:51:31,663 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/fe9b621d4c3c4f2c8dec6584c93fd07c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fe9b621d4c3c4f2c8dec6584c93fd07c 2024-11-16T14:51:31,674 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/877a9c4bc75c4a099fd320313f1f68c0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/877a9c4bc75c4a099fd320313f1f68c0 2024-11-16T14:51:31,681 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/B of c410da0e7ea4faea21a8b41f311130b9 into fe9b621d4c3c4f2c8dec6584c93fd07c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:31,681 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:31,681 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/B, priority=12, startTime=1731768691594; duration=0sec 2024-11-16T14:51:31,681 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:31,681 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:B 2024-11-16T14:51:31,681 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:31,687 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:31,688 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/C is initiating minor compaction (all files) 2024-11-16T14:51:31,688 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/C in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:31,688 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f242d70ae8554736bbd9828919928f9c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0a4592fb2de49a7b5157a86553608b7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/0a68f9d9fd4f409cb6fd80012aa0775a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/592e33cb967f4a0695eb72311d846fb3] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=48.2 K 2024-11-16T14:51:31,689 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f242d70ae8554736bbd9828919928f9c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768687128 2024-11-16T14:51:31,689 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a4592fb2de49a7b5157a86553608b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731768687765 2024-11-16T14:51:31,690 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a68f9d9fd4f409cb6fd80012aa0775a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731768688462 2024-11-16T14:51:31,691 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 592e33cb967f4a0695eb72311d846fb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731768690638 2024-11-16T14:51:31,694 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/A of c410da0e7ea4faea21a8b41f311130b9 into 877a9c4bc75c4a099fd320313f1f68c0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:31,695 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:31,695 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/A, priority=12, startTime=1731768691594; duration=0sec 2024-11-16T14:51:31,695 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:31,695 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:A 2024-11-16T14:51:31,710 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:31,711 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/7be52233a0924d698ab2d31668eb2efb is 50, key is test_row_0/C:col10/1731768690644/Put/seqid=0 2024-11-16T14:51:31,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741894_1070 (size=12915) 2024-11-16T14:51:31,740 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/7be52233a0924d698ab2d31668eb2efb as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/7be52233a0924d698ab2d31668eb2efb 2024-11-16T14:51:31,749 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/C of c410da0e7ea4faea21a8b41f311130b9 into 7be52233a0924d698ab2d31668eb2efb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:31,749 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:31,749 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/C, priority=12, startTime=1731768691594; duration=0sec 2024-11-16T14:51:31,750 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:31,750 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:C 2024-11-16T14:51:31,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:31,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:31,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768751893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768751896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768751894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768751896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768751897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768752011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768752011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768752011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768752020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768752021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,033 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/3ff2b0c436cf43f48df2d9dc099ce670 2024-11-16T14:51:32,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/f91d2590efff495e986228c61aec3821 is 50, key is test_row_0/B:col10/1731768690722/Put/seqid=0 2024-11-16T14:51:32,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741895_1071 (size=12301) 2024-11-16T14:51:32,056 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/f91d2590efff495e986228c61aec3821 2024-11-16T14:51:32,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/8dbc65569376419495e7088d4be06a40 is 50, key is test_row_0/C:col10/1731768690722/Put/seqid=0 2024-11-16T14:51:32,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741896_1072 (size=12301) 2024-11-16T14:51:32,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768752221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768752221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768752221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768752230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768752231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,484 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/8dbc65569376419495e7088d4be06a40 2024-11-16T14:51:32,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/3ff2b0c436cf43f48df2d9dc099ce670 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3ff2b0c436cf43f48df2d9dc099ce670 2024-11-16T14:51:32,502 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3ff2b0c436cf43f48df2d9dc099ce670, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T14:51:32,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/f91d2590efff495e986228c61aec3821 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f91d2590efff495e986228c61aec3821 2024-11-16T14:51:32,510 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f91d2590efff495e986228c61aec3821, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T14:51:32,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/8dbc65569376419495e7088d4be06a40 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8dbc65569376419495e7088d4be06a40 2024-11-16T14:51:32,519 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8dbc65569376419495e7088d4be06a40, entries=150, sequenceid=311, filesize=12.0 K 2024-11-16T14:51:32,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for c410da0e7ea4faea21a8b41f311130b9 in 924ms, sequenceid=311, compaction requested=false 2024-11-16T14:51:32,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:32,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:32,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-16T14:51:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-16T14:51:32,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-16T14:51:32,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0160 sec 2024-11-16T14:51:32,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.0260 sec 2024-11-16T14:51:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:32,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:51:32,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:32,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:32,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:32,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:32,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:32,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:32,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/c2b4b36cc4314d6cba1883ee6ca796a4 is 50, key is test_row_0/A:col10/1731768691892/Put/seqid=0 2024-11-16T14:51:32,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741897_1073 (size=14741) 2024-11-16T14:51:32,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768752570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768752572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768692562, max timestamp=1731768692576Num uniques 859; 2024-11-16T14:51:32,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:51:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768692562, max timestamp=1731768692576Num uniques 859; 2024-11-16T14:51:32,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:51:32,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768692562, max timestamp=1731768692576Num uniques 859; 2024-11-16T14:51:32,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:51:32,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768752619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768752619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768752620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T14:51:32,661 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:32,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-16T14:51:32,664 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:51:32,665 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:32,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:32,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768752691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768752691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768752740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768752741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768752741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:51:32,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T14:51:32,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:32,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:32,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:32,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:32,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:32,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768752910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768752911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768752950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768752951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768752951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:32,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/c2b4b36cc4314d6cba1883ee6ca796a4 2024-11-16T14:51:32,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/1aaec0fac25f4b768579f97db5fa0d35 is 50, key is test_row_0/B:col10/1731768691892/Put/seqid=0 2024-11-16T14:51:32,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T14:51:32,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:32,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:32,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:32,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:32,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:51:32,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741898_1074 (size=12301) 2024-11-16T14:51:32,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/1aaec0fac25f4b768579f97db5fa0d35 2024-11-16T14:51:33,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/00118992bc294a28bb1bfe6bd29b4ff1 is 50, key is test_row_0/C:col10/1731768691892/Put/seqid=0 2024-11-16T14:51:33,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741899_1075 (size=12301) 2024-11-16T14:51:33,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/00118992bc294a28bb1bfe6bd29b4ff1 2024-11-16T14:51:33,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/c2b4b36cc4314d6cba1883ee6ca796a4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/c2b4b36cc4314d6cba1883ee6ca796a4 2024-11-16T14:51:33,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/c2b4b36cc4314d6cba1883ee6ca796a4, entries=200, sequenceid=326, filesize=14.4 K 2024-11-16T14:51:33,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/1aaec0fac25f4b768579f97db5fa0d35 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/1aaec0fac25f4b768579f97db5fa0d35 2024-11-16T14:51:33,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/1aaec0fac25f4b768579f97db5fa0d35, entries=150, sequenceid=326, filesize=12.0 K 2024-11-16T14:51:33,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/00118992bc294a28bb1bfe6bd29b4ff1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/00118992bc294a28bb1bfe6bd29b4ff1 2024-11-16T14:51:33,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/00118992bc294a28bb1bfe6bd29b4ff1, entries=150, sequenceid=326, filesize=12.0 K 2024-11-16T14:51:33,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for c410da0e7ea4faea21a8b41f311130b9 in 519ms, sequenceid=326, compaction requested=true 2024-11-16T14:51:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:33,051 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:33,051 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:33,052 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:33,052 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:33,052 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/A is initiating minor compaction (all files) 2024-11-16T14:51:33,052 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/B is initiating minor compaction (all files) 2024-11-16T14:51:33,052 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/A in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:33,053 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/B in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:33,053 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/877a9c4bc75c4a099fd320313f1f68c0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3ff2b0c436cf43f48df2d9dc099ce670, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/c2b4b36cc4314d6cba1883ee6ca796a4] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=39.0 K 2024-11-16T14:51:33,053 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fe9b621d4c3c4f2c8dec6584c93fd07c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f91d2590efff495e986228c61aec3821, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/1aaec0fac25f4b768579f97db5fa0d35] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=36.6 K 2024-11-16T14:51:33,053 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9b621d4c3c4f2c8dec6584c93fd07c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731768690638 2024-11-16T14:51:33,053 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 877a9c4bc75c4a099fd320313f1f68c0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731768690638 2024-11-16T14:51:33,053 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f91d2590efff495e986228c61aec3821, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731768690717 2024-11-16T14:51:33,053 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ff2b0c436cf43f48df2d9dc099ce670, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731768690717 2024-11-16T14:51:33,054 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aaec0fac25f4b768579f97db5fa0d35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731768691892 2024-11-16T14:51:33,054 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b4b36cc4314d6cba1883ee6ca796a4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731768691890 2024-11-16T14:51:33,064 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#A#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:33,065 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/b9e249c05c854305af22d737a0be09fa is 50, key is test_row_0/A:col10/1731768691892/Put/seqid=0 2024-11-16T14:51:33,067 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:33,068 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/e14cb691cdcd472f99b460decf91a3cc is 50, key is test_row_0/B:col10/1731768691892/Put/seqid=0 2024-11-16T14:51:33,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741900_1076 (size=13017) 2024-11-16T14:51:33,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741901_1077 (size=13017) 2024-11-16T14:51:33,090 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/b9e249c05c854305af22d737a0be09fa as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b9e249c05c854305af22d737a0be09fa 2024-11-16T14:51:33,100 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/A of c410da0e7ea4faea21a8b41f311130b9 into b9e249c05c854305af22d737a0be09fa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:33,100 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:33,100 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/A, priority=13, startTime=1731768693051; duration=0sec 2024-11-16T14:51:33,100 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:33,101 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:A 2024-11-16T14:51:33,101 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:33,103 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:33,103 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/C is initiating minor compaction (all files) 2024-11-16T14:51:33,103 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/C in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:33,103 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/7be52233a0924d698ab2d31668eb2efb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8dbc65569376419495e7088d4be06a40, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/00118992bc294a28bb1bfe6bd29b4ff1] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=36.6 K 2024-11-16T14:51:33,103 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7be52233a0924d698ab2d31668eb2efb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731768690638 2024-11-16T14:51:33,104 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dbc65569376419495e7088d4be06a40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731768690717 2024-11-16T14:51:33,104 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00118992bc294a28bb1bfe6bd29b4ff1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731768691892 2024-11-16T14:51:33,113 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#C#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:33,114 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d55ac5ec3abf48739811f84533346ae6 is 50, key is test_row_0/C:col10/1731768691892/Put/seqid=0 2024-11-16T14:51:33,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741902_1078 (size=13017) 2024-11-16T14:51:33,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T14:51:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:33,128 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-11-16T14:51:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:33,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/8bd0c618600247a0a41d0e6ee65941cd is 50, key is test_row_0/A:col10/1731768692615/Put/seqid=0 2024-11-16T14:51:33,135 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d55ac5ec3abf48739811f84533346ae6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d55ac5ec3abf48739811f84533346ae6 2024-11-16T14:51:33,143 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/C of c410da0e7ea4faea21a8b41f311130b9 into d55ac5ec3abf48739811f84533346ae6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:33,143 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:33,143 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/C, priority=13, startTime=1731768693051; duration=0sec 2024-11-16T14:51:33,144 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:33,144 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:C 2024-11-16T14:51:33,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741903_1079 (size=12301) 2024-11-16T14:51:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:33,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:33,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768753244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768753245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768753261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768753261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768753261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:51:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768753350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768753361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,493 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/e14cb691cdcd472f99b460decf91a3cc as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/e14cb691cdcd472f99b460decf91a3cc 2024-11-16T14:51:33,518 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/B of c410da0e7ea4faea21a8b41f311130b9 into e14cb691cdcd472f99b460decf91a3cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:33,518 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:33,518 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/B, priority=13, startTime=1731768693051; duration=0sec 2024-11-16T14:51:33,518 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:33,518 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:B 2024-11-16T14:51:33,550 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/8bd0c618600247a0a41d0e6ee65941cd 2024-11-16T14:51:33,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768753561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/cc83671443ff4704900a648a98703ace is 50, key is test_row_0/B:col10/1731768692615/Put/seqid=0 2024-11-16T14:51:33,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768753571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741904_1080 (size=12301) 2024-11-16T14:51:33,592 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/cc83671443ff4704900a648a98703ace 2024-11-16T14:51:33,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/4b63d2c88892498b92e9e004e6c68494 is 50, key is test_row_0/C:col10/1731768692615/Put/seqid=0 2024-11-16T14:51:33,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741905_1081 (size=12301) 2024-11-16T14:51:33,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768753771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768753771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768753780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:51:33,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768753870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:33,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768753881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:34,029 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/4b63d2c88892498b92e9e004e6c68494 2024-11-16T14:51:34,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/8bd0c618600247a0a41d0e6ee65941cd as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8bd0c618600247a0a41d0e6ee65941cd 2024-11-16T14:51:34,048 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8bd0c618600247a0a41d0e6ee65941cd, entries=150, sequenceid=360, filesize=12.0 K 2024-11-16T14:51:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/cc83671443ff4704900a648a98703ace as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/cc83671443ff4704900a648a98703ace 2024-11-16T14:51:34,060 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/cc83671443ff4704900a648a98703ace, entries=150, sequenceid=360, filesize=12.0 K 2024-11-16T14:51:34,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/4b63d2c88892498b92e9e004e6c68494 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4b63d2c88892498b92e9e004e6c68494 2024-11-16T14:51:34,070 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4b63d2c88892498b92e9e004e6c68494, entries=150, sequenceid=360, filesize=12.0 K 2024-11-16T14:51:34,072 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=60.38 KB/61830 for c410da0e7ea4faea21a8b41f311130b9 in 944ms, sequenceid=360, compaction requested=false 2024-11-16T14:51:34,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:34,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:34,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-16T14:51:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-16T14:51:34,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-16T14:51:34,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-16T14:51:34,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4160 sec 2024-11-16T14:51:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:34,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:51:34,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:34,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:34,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:34,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:34,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:34,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:34,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/3552474408b44ca7b1eb7180bb9d10b1 is 50, key is test_row_0/A:col10/1731768693243/Put/seqid=0 2024-11-16T14:51:34,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741906_1082 (size=14741) 2024-11-16T14:51:34,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768754448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:34,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768754449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:34,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768754560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768754560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:34,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:34,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768754770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768754770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:34,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768754781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:34,783 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in c410da0e7ea4faea21a8b41f311130b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:34,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768754790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:34,795 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in c410da0e7ea4faea21a8b41f311130b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:34,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768754800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:34,801 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in c410da0e7ea4faea21a8b41f311130b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T14:51:34,811 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:34,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-16T14:51:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:51:34,814 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:34,814 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:34,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:34,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/3552474408b44ca7b1eb7180bb9d10b1 2024-11-16T14:51:34,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/2509c351baf64a1bb30111cc844951a4 is 50, key is test_row_0/B:col10/1731768693243/Put/seqid=0 2024-11-16T14:51:34,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741907_1083 (size=12301) 2024-11-16T14:51:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:51:34,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T14:51:34,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:34,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:34,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:34,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:34,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:35,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768755080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768755081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:35,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T14:51:35,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:35,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:35,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:35,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:35,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:51:35,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/2509c351baf64a1bb30111cc844951a4 2024-11-16T14:51:35,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/bfd8bf8596e547fa95f407a860c0c308 is 50, key is test_row_0/C:col10/1731768693243/Put/seqid=0 2024-11-16T14:51:35,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741908_1084 (size=12301) 2024-11-16T14:51:35,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/bfd8bf8596e547fa95f407a860c0c308 2024-11-16T14:51:35,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T14:51:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:35,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:35,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/3552474408b44ca7b1eb7180bb9d10b1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3552474408b44ca7b1eb7180bb9d10b1 2024-11-16T14:51:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:35,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3552474408b44ca7b1eb7180bb9d10b1, entries=200, sequenceid=375, filesize=14.4 K 2024-11-16T14:51:35,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/2509c351baf64a1bb30111cc844951a4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/2509c351baf64a1bb30111cc844951a4 2024-11-16T14:51:35,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/2509c351baf64a1bb30111cc844951a4, entries=150, sequenceid=375, filesize=12.0 K 2024-11-16T14:51:35,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/bfd8bf8596e547fa95f407a860c0c308 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/bfd8bf8596e547fa95f407a860c0c308 2024-11-16T14:51:35,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/bfd8bf8596e547fa95f407a860c0c308, entries=150, sequenceid=375, filesize=12.0 K 2024-11-16T14:51:35,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c410da0e7ea4faea21a8b41f311130b9 in 915ms, sequenceid=375, compaction requested=true 2024-11-16T14:51:35,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:35,303 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:35,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:35,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:35,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:35,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:35,305 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:35,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:35,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:35,306 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:35,306 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/A is initiating minor compaction (all files) 2024-11-16T14:51:35,306 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/A in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:35,306 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b9e249c05c854305af22d737a0be09fa, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8bd0c618600247a0a41d0e6ee65941cd, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3552474408b44ca7b1eb7180bb9d10b1] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=39.1 K 2024-11-16T14:51:35,307 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e249c05c854305af22d737a0be09fa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731768691892 2024-11-16T14:51:35,307 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:35,307 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/B is initiating minor compaction (all files) 2024-11-16T14:51:35,307 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/B in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:35,307 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/e14cb691cdcd472f99b460decf91a3cc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/cc83671443ff4704900a648a98703ace, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/2509c351baf64a1bb30111cc844951a4] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=36.7 K 2024-11-16T14:51:35,308 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bd0c618600247a0a41d0e6ee65941cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731768692610 2024-11-16T14:51:35,308 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting e14cb691cdcd472f99b460decf91a3cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731768691892 2024-11-16T14:51:35,308 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3552474408b44ca7b1eb7180bb9d10b1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731768693223 2024-11-16T14:51:35,308 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting cc83671443ff4704900a648a98703ace, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731768692610 2024-11-16T14:51:35,309 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2509c351baf64a1bb30111cc844951a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731768693223 2024-11-16T14:51:35,341 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:35,341 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/81b40b75d3b54f018dfe71b55749cd40 is 50, key is test_row_0/B:col10/1731768693243/Put/seqid=0 2024-11-16T14:51:35,344 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:35,345 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/d98639f3210a4e009319130f3b467c6e is 50, key is test_row_0/A:col10/1731768693243/Put/seqid=0 2024-11-16T14:51:35,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741910_1086 (size=13119) 2024-11-16T14:51:35,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741909_1085 (size=13119) 2024-11-16T14:51:35,371 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/d98639f3210a4e009319130f3b467c6e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/d98639f3210a4e009319130f3b467c6e 2024-11-16T14:51:35,383 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/A of c410da0e7ea4faea21a8b41f311130b9 into d98639f3210a4e009319130f3b467c6e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:35,383 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:35,383 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/A, priority=13, startTime=1731768695303; duration=0sec 2024-11-16T14:51:35,383 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:35,384 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:A 2024-11-16T14:51:35,384 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:35,386 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:35,386 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/C is initiating minor compaction (all files) 2024-11-16T14:51:35,386 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/C in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:35,386 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d55ac5ec3abf48739811f84533346ae6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4b63d2c88892498b92e9e004e6c68494, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/bfd8bf8596e547fa95f407a860c0c308] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=36.7 K 2024-11-16T14:51:35,387 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d55ac5ec3abf48739811f84533346ae6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731768691892 2024-11-16T14:51:35,387 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b63d2c88892498b92e9e004e6c68494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731768692610 2024-11-16T14:51:35,387 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfd8bf8596e547fa95f407a860c0c308, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731768693223 2024-11-16T14:51:35,399 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:35,400 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/c82e61ea0c324529807013260a935aed is 50, key is test_row_0/C:col10/1731768693243/Put/seqid=0 2024-11-16T14:51:35,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T14:51:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:35,427 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:51:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:35,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741911_1087 (size=13119) 2024-11-16T14:51:35,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/79bda35388004d529147d3b9b413de99 is 50, key is test_row_0/A:col10/1731768694432/Put/seqid=0 2024-11-16T14:51:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:51:35,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/c82e61ea0c324529807013260a935aed as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/c82e61ea0c324529807013260a935aed 2024-11-16T14:51:35,452 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/C of c410da0e7ea4faea21a8b41f311130b9 into c82e61ea0c324529807013260a935aed(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:35,452 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:35,452 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/C, priority=13, startTime=1731768695305; duration=0sec 2024-11-16T14:51:35,453 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:35,453 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:C 2024-11-16T14:51:35,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741912_1088 (size=12301) 2024-11-16T14:51:35,463 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/79bda35388004d529147d3b9b413de99 2024-11-16T14:51:35,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/9194476557014e99b698ec86b3f44e69 is 50, key is test_row_0/B:col10/1731768694432/Put/seqid=0 2024-11-16T14:51:35,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741913_1089 (size=12301) 2024-11-16T14:51:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:35,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:35,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768755607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768755608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:35,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768755721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768755721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:35,764 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/81b40b75d3b54f018dfe71b55749cd40 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/81b40b75d3b54f018dfe71b55749cd40 2024-11-16T14:51:35,770 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/B of c410da0e7ea4faea21a8b41f311130b9 into 81b40b75d3b54f018dfe71b55749cd40(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:35,771 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:35,771 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/B, priority=13, startTime=1731768695305; duration=0sec 2024-11-16T14:51:35,771 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:35,771 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:B 2024-11-16T14:51:35,889 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/9194476557014e99b698ec86b3f44e69 2024-11-16T14:51:35,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/2b0ea986dfaa499c967ee055152e714b is 50, key is test_row_0/C:col10/1731768694432/Put/seqid=0 2024-11-16T14:51:35,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741914_1090 (size=12301) 2024-11-16T14:51:35,908 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/2b0ea986dfaa499c967ee055152e714b 2024-11-16T14:51:35,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/79bda35388004d529147d3b9b413de99 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/79bda35388004d529147d3b9b413de99 2024-11-16T14:51:35,923 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/79bda35388004d529147d3b9b413de99, entries=150, sequenceid=395, filesize=12.0 K 2024-11-16T14:51:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/9194476557014e99b698ec86b3f44e69 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9194476557014e99b698ec86b3f44e69 2024-11-16T14:51:35,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768755931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:35,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,933 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9194476557014e99b698ec86b3f44e69, entries=150, sequenceid=395, filesize=12.0 K 2024-11-16T14:51:35,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:35,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768755932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:35,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:35,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:35,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/2b0ea986dfaa499c967ee055152e714b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/2b0ea986dfaa499c967ee055152e714b 2024-11-16T14:51:35,943 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/2b0ea986dfaa499c967ee055152e714b, entries=150, sequenceid=395, filesize=12.0 K 2024-11-16T14:51:35,946 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c410da0e7ea4faea21a8b41f311130b9 in 518ms, sequenceid=395, compaction requested=false 2024-11-16T14:51:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-16T14:51:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-16T14:51:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:51:35,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-16T14:51:35,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1330 sec 2024-11-16T14:51:35,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1430 sec 2024-11-16T14:51:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:36,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:51:36,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:36,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:36,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:36,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:36,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:36,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:36,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/186cbe74c77e4c229c4a48d367085f4d is 50, key is test_row_0/A:col10/1731768695603/Put/seqid=0 2024-11-16T14:51:36,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741915_1091 (size=12301) 2024-11-16T14:51:36,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768756275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768756276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768756381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768756381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768756591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768756591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/186cbe74c77e4c229c4a48d367085f4d 2024-11-16T14:51:36,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/f45c9ae7c5674088b765c79bf05ece0c is 50, key is test_row_0/B:col10/1731768695603/Put/seqid=0 2024-11-16T14:51:36,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741916_1092 (size=12301) 2024-11-16T14:51:36,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/f45c9ae7c5674088b765c79bf05ece0c 2024-11-16T14:51:36,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/48a6723c34164f8c8ed3b073c74a74fe is 50, key is test_row_0/C:col10/1731768695603/Put/seqid=0 2024-11-16T14:51:36,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741917_1093 (size=12301) 2024-11-16T14:51:36,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59034 deadline: 1731768756811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58986 deadline: 1731768756811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,814 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in c410da0e7ea4faea21a8b41f311130b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,815 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in c410da0e7ea4faea21a8b41f311130b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59018 deadline: 1731768756830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,833 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in c410da0e7ea4faea21a8b41f311130b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:36,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768756900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768756900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T14:51:36,961 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:36,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-16T14:51:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:51:36,965 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:36,965 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:36,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:51:37,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/48a6723c34164f8c8ed3b073c74a74fe 2024-11-16T14:51:37,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/186cbe74c77e4c229c4a48d367085f4d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/186cbe74c77e4c229c4a48d367085f4d 2024-11-16T14:51:37,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/186cbe74c77e4c229c4a48d367085f4d, entries=150, sequenceid=411, filesize=12.0 K 2024-11-16T14:51:37,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/f45c9ae7c5674088b765c79bf05ece0c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f45c9ae7c5674088b765c79bf05ece0c 2024-11-16T14:51:37,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f45c9ae7c5674088b765c79bf05ece0c, entries=150, sequenceid=411, filesize=12.0 K 2024-11-16T14:51:37,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/48a6723c34164f8c8ed3b073c74a74fe as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/48a6723c34164f8c8ed3b073c74a74fe 2024-11-16T14:51:37,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/48a6723c34164f8c8ed3b073c74a74fe, entries=150, sequenceid=411, filesize=12.0 K 2024-11-16T14:51:37,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c410da0e7ea4faea21a8b41f311130b9 in 869ms, sequenceid=411, compaction requested=true 2024-11-16T14:51:37,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:37,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:37,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:37,112 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:37,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:37,112 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:37,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:37,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:37,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:37,113 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:37,113 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/B is initiating minor compaction (all files) 2024-11-16T14:51:37,113 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/B in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:37,113 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/81b40b75d3b54f018dfe71b55749cd40, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9194476557014e99b698ec86b3f44e69, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f45c9ae7c5674088b765c79bf05ece0c] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=36.8 K 2024-11-16T14:51:37,114 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:37,114 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/A is initiating minor compaction (all files) 2024-11-16T14:51:37,114 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/A in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:37,114 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/d98639f3210a4e009319130f3b467c6e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/79bda35388004d529147d3b9b413de99, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/186cbe74c77e4c229c4a48d367085f4d] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=36.8 K 2024-11-16T14:51:37,114 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b40b75d3b54f018dfe71b55749cd40, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731768693223 2024-11-16T14:51:37,115 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d98639f3210a4e009319130f3b467c6e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731768693223 2024-11-16T14:51:37,115 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9194476557014e99b698ec86b3f44e69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731768694432 2024-11-16T14:51:37,115 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79bda35388004d529147d3b9b413de99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731768694432 2024-11-16T14:51:37,115 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f45c9ae7c5674088b765c79bf05ece0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731768695602 2024-11-16T14:51:37,116 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 186cbe74c77e4c229c4a48d367085f4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731768695602 2024-11-16T14:51:37,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T14:51:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:37,119 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T14:51:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:37,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/aa9ddbf701a640d89092fa5949afc652 is 50, key is test_row_0/A:col10/1731768696271/Put/seqid=0 2024-11-16T14:51:37,128 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:37,128 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/0b2843da4f4c4fbcb7315d1b6ed5d106 is 50, key is test_row_0/A:col10/1731768695603/Put/seqid=0 2024-11-16T14:51:37,132 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#B#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:37,133 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/42d6c4cd554b47a39dc910a9e8624fcf is 50, key is test_row_0/B:col10/1731768695603/Put/seqid=0 2024-11-16T14:51:37,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741918_1094 (size=12301) 2024-11-16T14:51:37,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741919_1095 (size=13221) 2024-11-16T14:51:37,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741920_1096 (size=13221) 2024-11-16T14:51:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:51:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:37,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768757436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:37,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768757440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:37,536 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/aa9ddbf701a640d89092fa5949afc652 2024-11-16T14:51:37,545 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/0b2843da4f4c4fbcb7315d1b6ed5d106 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/0b2843da4f4c4fbcb7315d1b6ed5d106 2024-11-16T14:51:37,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/fa99cddb32a54500a4b1a79757a79fd6 is 50, key is test_row_0/B:col10/1731768696271/Put/seqid=0 2024-11-16T14:51:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768757551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768757551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:37,553 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/A of c410da0e7ea4faea21a8b41f311130b9 into 0b2843da4f4c4fbcb7315d1b6ed5d106(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:37,553 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:37,553 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/A, priority=13, startTime=1731768697111; duration=0sec 2024-11-16T14:51:37,553 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:37,553 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:A 2024-11-16T14:51:37,553 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:37,555 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:37,556 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/C is initiating minor compaction (all files) 2024-11-16T14:51:37,556 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/C in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:37,556 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/c82e61ea0c324529807013260a935aed, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/2b0ea986dfaa499c967ee055152e714b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/48a6723c34164f8c8ed3b073c74a74fe] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=36.8 K 2024-11-16T14:51:37,558 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c82e61ea0c324529807013260a935aed, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731768693223 2024-11-16T14:51:37,558 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/42d6c4cd554b47a39dc910a9e8624fcf as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/42d6c4cd554b47a39dc910a9e8624fcf 2024-11-16T14:51:37,558 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b0ea986dfaa499c967ee055152e714b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731768694432 2024-11-16T14:51:37,559 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a6723c34164f8c8ed3b073c74a74fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731768695602 2024-11-16T14:51:37,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741921_1097 (size=12301) 2024-11-16T14:51:37,560 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/fa99cddb32a54500a4b1a79757a79fd6 2024-11-16T14:51:37,566 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/B of c410da0e7ea4faea21a8b41f311130b9 into 42d6c4cd554b47a39dc910a9e8624fcf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:37,567 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:37,567 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/B, priority=13, startTime=1731768697112; duration=0sec 2024-11-16T14:51:37,567 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:37,567 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:B 2024-11-16T14:51:37,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/09fa8a40cef3495fb596534df75c477c is 50, key is test_row_0/C:col10/1731768696271/Put/seqid=0 2024-11-16T14:51:37,575 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:37,576 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/beeda26eb1474c13824f72f9206a0387 is 50, key is test_row_0/C:col10/1731768695603/Put/seqid=0 2024-11-16T14:51:37,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741922_1098 (size=12301) 2024-11-16T14:51:37,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741923_1099 (size=13221) 2024-11-16T14:51:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:51:37,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768757761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:37,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768757761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:37,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:37,985 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/09fa8a40cef3495fb596534df75c477c 2024-11-16T14:51:37,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/aa9ddbf701a640d89092fa5949afc652 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/aa9ddbf701a640d89092fa5949afc652 2024-11-16T14:51:37,995 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/beeda26eb1474c13824f72f9206a0387 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/beeda26eb1474c13824f72f9206a0387 2024-11-16T14:51:37,999 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/aa9ddbf701a640d89092fa5949afc652, entries=150, sequenceid=429, filesize=12.0 K 2024-11-16T14:51:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/fa99cddb32a54500a4b1a79757a79fd6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fa99cddb32a54500a4b1a79757a79fd6 2024-11-16T14:51:38,002 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/C of c410da0e7ea4faea21a8b41f311130b9 into beeda26eb1474c13824f72f9206a0387(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:38,002 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:38,002 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/C, priority=13, startTime=1731768697112; duration=0sec 2024-11-16T14:51:38,002 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:38,002 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:C 2024-11-16T14:51:38,007 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fa99cddb32a54500a4b1a79757a79fd6, entries=150, sequenceid=429, filesize=12.0 K 2024-11-16T14:51:38,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/09fa8a40cef3495fb596534df75c477c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/09fa8a40cef3495fb596534df75c477c 2024-11-16T14:51:38,013 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/09fa8a40cef3495fb596534df75c477c, entries=150, sequenceid=429, filesize=12.0 K 2024-11-16T14:51:38,014 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c410da0e7ea4faea21a8b41f311130b9 in 895ms, sequenceid=429, compaction requested=false 2024-11-16T14:51:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-16T14:51:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-16T14:51:38,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-16T14:51:38,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0500 sec 2024-11-16T14:51:38,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.0560 sec 2024-11-16T14:51:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:38,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T14:51:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:38,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/a60f5c2e5df5449ca574b818602444a9 is 50, key is test_row_0/A:col10/1731768698073/Put/seqid=0 2024-11-16T14:51:38,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741924_1100 (size=17181) 2024-11-16T14:51:38,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/a60f5c2e5df5449ca574b818602444a9 2024-11-16T14:51:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T14:51:38,100 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:38,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-16T14:51:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:51:38,104 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:38,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/776005a442d2461fa2a93e0c098bde68 is 50, key is test_row_0/B:col10/1731768698073/Put/seqid=0 2024-11-16T14:51:38,105 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:38,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:38,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768758112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:38,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768758118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:38,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:38,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741925_1101 (size=12301) 2024-11-16T14:51:38,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/776005a442d2461fa2a93e0c098bde68 2024-11-16T14:51:38,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d9e4dbe754a64953adc22a19a60b78e5 is 50, key is test_row_0/C:col10/1731768698073/Put/seqid=0 2024-11-16T14:51:38,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741926_1102 (size=12301) 2024-11-16T14:51:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:51:38,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768758220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:38,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768758230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:38,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:38,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T14:51:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:38,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:51:38,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T14:51:38,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:38,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:38,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768758430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:38,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768758441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:38,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d9e4dbe754a64953adc22a19a60b78e5 2024-11-16T14:51:38,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/a60f5c2e5df5449ca574b818602444a9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/a60f5c2e5df5449ca574b818602444a9 2024-11-16T14:51:38,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/a60f5c2e5df5449ca574b818602444a9, entries=250, sequenceid=448, filesize=16.8 K 2024-11-16T14:51:38,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/776005a442d2461fa2a93e0c098bde68 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/776005a442d2461fa2a93e0c098bde68 2024-11-16T14:51:38,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/776005a442d2461fa2a93e0c098bde68, entries=150, sequenceid=448, filesize=12.0 K 2024-11-16T14:51:38,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d9e4dbe754a64953adc22a19a60b78e5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d9e4dbe754a64953adc22a19a60b78e5 2024-11-16T14:51:38,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d9e4dbe754a64953adc22a19a60b78e5, entries=150, sequenceid=448, filesize=12.0 K 2024-11-16T14:51:38,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c410da0e7ea4faea21a8b41f311130b9 in 504ms, sequenceid=448, compaction requested=true 2024-11-16T14:51:38,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:38,577 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:38,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:38,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:38,578 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:38,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:38,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:38,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:38,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:38,579 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:38,579 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/A is initiating minor compaction (all files) 2024-11-16T14:51:38,579 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/A in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:38,579 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/0b2843da4f4c4fbcb7315d1b6ed5d106, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/aa9ddbf701a640d89092fa5949afc652, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/a60f5c2e5df5449ca574b818602444a9] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=41.7 K 2024-11-16T14:51:38,580 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b2843da4f4c4fbcb7315d1b6ed5d106, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731768695602 2024-11-16T14:51:38,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T14:51:38,582 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:38,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:38,582 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/B is initiating minor compaction (all files) 2024-11-16T14:51:38,582 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/B in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:38,582 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T14:51:38,582 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/42d6c4cd554b47a39dc910a9e8624fcf, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fa99cddb32a54500a4b1a79757a79fd6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/776005a442d2461fa2a93e0c098bde68] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=36.9 K 2024-11-16T14:51:38,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:38,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:38,582 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9ddbf701a640d89092fa5949afc652, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731768696271 2024-11-16T14:51:38,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:38,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:38,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:38,583 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 42d6c4cd554b47a39dc910a9e8624fcf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731768695602 2024-11-16T14:51:38,583 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting fa99cddb32a54500a4b1a79757a79fd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731768696271 2024-11-16T14:51:38,584 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a60f5c2e5df5449ca574b818602444a9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731768697429 2024-11-16T14:51:38,584 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 776005a442d2461fa2a93e0c098bde68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731768697434 2024-11-16T14:51:38,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/60fc4178224a4c13becb7e8252c38bc2 is 50, key is test_row_0/A:col10/1731768698102/Put/seqid=0 2024-11-16T14:51:38,594 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:38,595 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/d36f5d5a72a84d38b8e5f493ef1fc1e9 is 50, key is test_row_0/B:col10/1731768698073/Put/seqid=0 2024-11-16T14:51:38,612 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#A#compaction#92 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:38,613 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/fd84e0dc646840d18af851df30ce5564 is 50, key is test_row_0/A:col10/1731768698073/Put/seqid=0 2024-11-16T14:51:38,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741927_1103 (size=12301) 2024-11-16T14:51:38,626 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/60fc4178224a4c13becb7e8252c38bc2 2024-11-16T14:51:38,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741928_1104 (size=13323) 2024-11-16T14:51:38,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741929_1105 (size=13323) 2024-11-16T14:51:38,660 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/d36f5d5a72a84d38b8e5f493ef1fc1e9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/d36f5d5a72a84d38b8e5f493ef1fc1e9 2024-11-16T14:51:38,668 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/fd84e0dc646840d18af851df30ce5564 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/fd84e0dc646840d18af851df30ce5564 2024-11-16T14:51:38,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/4feb35c9e0a446988014bd28d4b46656 is 50, key is test_row_0/B:col10/1731768698102/Put/seqid=0 2024-11-16T14:51:38,676 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/B of c410da0e7ea4faea21a8b41f311130b9 into d36f5d5a72a84d38b8e5f493ef1fc1e9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:38,676 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:38,676 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/B, priority=13, startTime=1731768698578; duration=0sec 2024-11-16T14:51:38,676 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:38,676 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:B 2024-11-16T14:51:38,676 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:38,679 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:38,679 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/C is initiating minor compaction (all files) 2024-11-16T14:51:38,679 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/C in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:38,679 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/beeda26eb1474c13824f72f9206a0387, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/09fa8a40cef3495fb596534df75c477c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d9e4dbe754a64953adc22a19a60b78e5] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=36.9 K 2024-11-16T14:51:38,680 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting beeda26eb1474c13824f72f9206a0387, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731768695602 2024-11-16T14:51:38,681 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fa8a40cef3495fb596534df75c477c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731768696271 2024-11-16T14:51:38,681 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d9e4dbe754a64953adc22a19a60b78e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731768697434 2024-11-16T14:51:38,684 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/A of c410da0e7ea4faea21a8b41f311130b9 into fd84e0dc646840d18af851df30ce5564(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:38,684 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:38,684 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/A, priority=13, startTime=1731768698577; duration=0sec 2024-11-16T14:51:38,684 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:38,684 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:A 2024-11-16T14:51:38,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741930_1106 (size=12301) 2024-11-16T14:51:38,705 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/4feb35c9e0a446988014bd28d4b46656 2024-11-16T14:51:38,709 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#C#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:38,710 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/575cd445787046fb88db840d47ac1973 is 50, key is test_row_0/C:col10/1731768698073/Put/seqid=0 2024-11-16T14:51:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:51:38,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/73d112d1ddf24903bc6e26d6e4b78143 is 50, key is test_row_0/C:col10/1731768698102/Put/seqid=0 2024-11-16T14:51:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:38,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:38,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741931_1107 (size=13323) 2024-11-16T14:51:38,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768758776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:38,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768758777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:38,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741932_1108 (size=12301) 2024-11-16T14:51:38,790 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/73d112d1ddf24903bc6e26d6e4b78143 2024-11-16T14:51:38,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/60fc4178224a4c13becb7e8252c38bc2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/60fc4178224a4c13becb7e8252c38bc2 2024-11-16T14:51:38,804 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/60fc4178224a4c13becb7e8252c38bc2, entries=150, sequenceid=464, filesize=12.0 K 2024-11-16T14:51:38,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/4feb35c9e0a446988014bd28d4b46656 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4feb35c9e0a446988014bd28d4b46656 2024-11-16T14:51:38,814 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4feb35c9e0a446988014bd28d4b46656, entries=150, sequenceid=464, filesize=12.0 K 2024-11-16T14:51:38,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/73d112d1ddf24903bc6e26d6e4b78143 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/73d112d1ddf24903bc6e26d6e4b78143 2024-11-16T14:51:38,826 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/73d112d1ddf24903bc6e26d6e4b78143, entries=150, sequenceid=464, filesize=12.0 K 2024-11-16T14:51:38,828 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c410da0e7ea4faea21a8b41f311130b9 in 246ms, sequenceid=464, compaction requested=false 2024-11-16T14:51:38,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:38,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:38,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-16T14:51:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-16T14:51:38,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-16T14:51:38,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 725 msec 2024-11-16T14:51:38,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 734 msec 2024-11-16T14:51:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:38,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T14:51:38,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:38,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:38,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:38,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:38,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:38,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:38,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/61548f29988b444bb1f3bda7ae850310 is 50, key is test_row_0/A:col10/1731768698765/Put/seqid=0 2024-11-16T14:51:38,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741933_1109 (size=14741) 2024-11-16T14:51:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768758959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:38,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768758959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:39,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768759071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768759071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:39,163 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/575cd445787046fb88db840d47ac1973 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/575cd445787046fb88db840d47ac1973 2024-11-16T14:51:39,171 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/C of c410da0e7ea4faea21a8b41f311130b9 into 575cd445787046fb88db840d47ac1973(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:39,171 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:39,171 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/C, priority=13, startTime=1731768698579; duration=0sec 2024-11-16T14:51:39,171 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:39,171 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:C 2024-11-16T14:51:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T14:51:39,241 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:39,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-16T14:51:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:51:39,243 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:39,244 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:39,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:39,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768759281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:39,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768759281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:39,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/61548f29988b444bb1f3bda7ae850310 2024-11-16T14:51:39,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/835b60ca773e4c7586a75d18c0f6e627 is 50, key is test_row_0/B:col10/1731768698765/Put/seqid=0 2024-11-16T14:51:39,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741934_1110 (size=12301) 2024-11-16T14:51:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:51:39,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T14:51:39,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:39,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:39,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:39,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:39,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:39,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T14:51:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:39,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:51:39,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59052 deadline: 1731768759591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:39,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1731768759591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:51:39,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T14:51:39,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:39,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:39,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:39,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:39,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:39,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/835b60ca773e4c7586a75d18c0f6e627 2024-11-16T14:51:39,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d0cf51ffb4e74e4cb71becf537ebb4b2 is 50, key is test_row_0/C:col10/1731768698765/Put/seqid=0 2024-11-16T14:51:39,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741935_1111 (size=12301) 2024-11-16T14:51:39,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d0cf51ffb4e74e4cb71becf537ebb4b2 2024-11-16T14:51:39,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/61548f29988b444bb1f3bda7ae850310 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/61548f29988b444bb1f3bda7ae850310 2024-11-16T14:51:39,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/61548f29988b444bb1f3bda7ae850310, entries=200, sequenceid=483, filesize=14.4 K 2024-11-16T14:51:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/835b60ca773e4c7586a75d18c0f6e627 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/835b60ca773e4c7586a75d18c0f6e627 2024-11-16T14:51:39,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/835b60ca773e4c7586a75d18c0f6e627, entries=150, sequenceid=483, filesize=12.0 K 2024-11-16T14:51:39,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/d0cf51ffb4e74e4cb71becf537ebb4b2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0cf51ffb4e74e4cb71becf537ebb4b2 2024-11-16T14:51:39,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0cf51ffb4e74e4cb71becf537ebb4b2, entries=150, sequenceid=483, filesize=12.0 K 2024-11-16T14:51:39,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for c410da0e7ea4faea21a8b41f311130b9 in 942ms, sequenceid=483, compaction requested=true 2024-11-16T14:51:39,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:39,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:39,835 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:39,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:39,835 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:39,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:39,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:39,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c410da0e7ea4faea21a8b41f311130b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:39,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:39,837 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:39,837 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:39,837 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/B is initiating minor compaction (all files) 2024-11-16T14:51:39,837 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/A is initiating minor compaction (all files) 2024-11-16T14:51:39,837 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/B in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:39,837 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/A in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:39,837 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/d36f5d5a72a84d38b8e5f493ef1fc1e9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4feb35c9e0a446988014bd28d4b46656, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/835b60ca773e4c7586a75d18c0f6e627] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=37.0 K 2024-11-16T14:51:39,837 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/fd84e0dc646840d18af851df30ce5564, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/60fc4178224a4c13becb7e8252c38bc2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/61548f29988b444bb1f3bda7ae850310] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=39.4 K 2024-11-16T14:51:39,837 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd84e0dc646840d18af851df30ce5564, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731768697434 2024-11-16T14:51:39,837 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d36f5d5a72a84d38b8e5f493ef1fc1e9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731768697434 2024-11-16T14:51:39,838 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60fc4178224a4c13becb7e8252c38bc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731768698094 2024-11-16T14:51:39,838 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4feb35c9e0a446988014bd28d4b46656, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731768698094 2024-11-16T14:51:39,838 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61548f29988b444bb1f3bda7ae850310, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731768698765 2024-11-16T14:51:39,838 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 835b60ca773e4c7586a75d18c0f6e627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731768698765 2024-11-16T14:51:39,853 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:39,853 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/e26750c6f3d94093ad59e3e927b4209c is 50, key is test_row_0/A:col10/1731768698765/Put/seqid=0 2024-11-16T14:51:39,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T14:51:39,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:39,862 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T14:51:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:39,863 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#B#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:39,864 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/edb7d2b1bdb7407eb2907b499980f5e4 is 50, key is test_row_0/B:col10/1731768698765/Put/seqid=0 2024-11-16T14:51:39,870 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-16T14:51:39,870 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:51:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:51:39,873 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-16T14:51:39,873 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:51:39,875 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-16T14:51:39,875 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:51:39,876 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:39,876 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:39,877 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:39,877 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:39,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:51:39,877 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:39,877 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:39,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:51:39,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:51:39,879 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-16T14:51:39,879 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:51:39,879 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:39,879 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:39,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:51:39,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/720a1767485a474787038a47a6f64252 is 50, key is test_row_0/A:col10/1731768698917/Put/seqid=0 2024-11-16T14:51:39,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741936_1112 (size=13425) 2024-11-16T14:51:39,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741937_1113 (size=13425) 2024-11-16T14:51:39,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741938_1114 (size=12301) 2024-11-16T14:51:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:40,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. as already flushing 2024-11-16T14:51:40,102 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-16T14:51:40,102 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:51:40,102 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:40,103 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:40,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:51:40,111 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-16T14:51:40,111 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:51:40,112 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:40,112 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:40,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:51:40,293 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/720a1767485a474787038a47a6f64252 2024-11-16T14:51:40,294 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/e26750c6f3d94093ad59e3e927b4209c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/e26750c6f3d94093ad59e3e927b4209c 2024-11-16T14:51:40,294 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/edb7d2b1bdb7407eb2907b499980f5e4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/edb7d2b1bdb7407eb2907b499980f5e4 2024-11-16T14:51:40,300 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/A of c410da0e7ea4faea21a8b41f311130b9 into e26750c6f3d94093ad59e3e927b4209c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:40,300 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:40,300 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/A, priority=13, startTime=1731768699835; duration=0sec 2024-11-16T14:51:40,300 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/B of c410da0e7ea4faea21a8b41f311130b9 into edb7d2b1bdb7407eb2907b499980f5e4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:40,300 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:40,300 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/B, priority=13, startTime=1731768699835; duration=0sec 2024-11-16T14:51:40,300 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:40,300 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:A 2024-11-16T14:51:40,300 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:40,300 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:40,300 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:B 2024-11-16T14:51:40,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/f8394085f4094863a1f358c14d9f3dc1 is 50, key is test_row_0/B:col10/1731768698917/Put/seqid=0 2024-11-16T14:51:40,301 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:40,301 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): c410da0e7ea4faea21a8b41f311130b9/C is initiating minor compaction (all files) 2024-11-16T14:51:40,302 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c410da0e7ea4faea21a8b41f311130b9/C in TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:40,302 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/575cd445787046fb88db840d47ac1973, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/73d112d1ddf24903bc6e26d6e4b78143, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0cf51ffb4e74e4cb71becf537ebb4b2] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp, totalSize=37.0 K 2024-11-16T14:51:40,302 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 575cd445787046fb88db840d47ac1973, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731768697434 2024-11-16T14:51:40,302 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73d112d1ddf24903bc6e26d6e4b78143, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731768698094 2024-11-16T14:51:40,303 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0cf51ffb4e74e4cb71becf537ebb4b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731768698765 2024-11-16T14:51:40,312 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c410da0e7ea4faea21a8b41f311130b9#C#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:40,312 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/dd031843e5c74b549b5294589b17f453 is 50, key is test_row_0/C:col10/1731768698765/Put/seqid=0 2024-11-16T14:51:40,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741939_1115 (size=12301) 2024-11-16T14:51:40,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741940_1116 (size=13425) 2024-11-16T14:51:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:51:40,715 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/f8394085f4094863a1f358c14d9f3dc1 2024-11-16T14:51:40,721 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/dd031843e5c74b549b5294589b17f453 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/dd031843e5c74b549b5294589b17f453 2024-11-16T14:51:40,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/23fd866e6c7a41bb9273b6f1883b6a3b is 50, key is test_row_0/C:col10/1731768698917/Put/seqid=0 2024-11-16T14:51:40,727 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c410da0e7ea4faea21a8b41f311130b9/C of c410da0e7ea4faea21a8b41f311130b9 into dd031843e5c74b549b5294589b17f453(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:40,727 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:40,727 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., storeName=c410da0e7ea4faea21a8b41f311130b9/C, priority=13, startTime=1731768699835; duration=0sec 2024-11-16T14:51:40,728 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:40,728 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c410da0e7ea4faea21a8b41f311130b9:C 2024-11-16T14:51:40,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741941_1117 (size=12301) 2024-11-16T14:51:40,862 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-16T14:51:40,862 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-16T14:51:40,862 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-16T14:51:40,863 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:51:40,863 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:51:40,863 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:51:40,863 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:40,863 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:40,863 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:40,863 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:40,863 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:40,863 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:40,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:51:40,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:51:40,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:51:41,129 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/23fd866e6c7a41bb9273b6f1883b6a3b 2024-11-16T14:51:41,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/720a1767485a474787038a47a6f64252 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/720a1767485a474787038a47a6f64252 2024-11-16T14:51:41,138 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/720a1767485a474787038a47a6f64252, entries=150, sequenceid=500, filesize=12.0 K 2024-11-16T14:51:41,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/f8394085f4094863a1f358c14d9f3dc1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f8394085f4094863a1f358c14d9f3dc1 2024-11-16T14:51:41,144 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f8394085f4094863a1f358c14d9f3dc1, entries=150, sequenceid=500, filesize=12.0 K 2024-11-16T14:51:41,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/23fd866e6c7a41bb9273b6f1883b6a3b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/23fd866e6c7a41bb9273b6f1883b6a3b 2024-11-16T14:51:41,150 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/23fd866e6c7a41bb9273b6f1883b6a3b, entries=150, sequenceid=500, filesize=12.0 K 2024-11-16T14:51:41,151 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=33.54 KB/34350 for c410da0e7ea4faea21a8b41f311130b9 in 1288ms, sequenceid=500, compaction requested=false 2024-11-16T14:51:41,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for c410da0e7ea4faea21a8b41f311130b9: 2024-11-16T14:51:41,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:41,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-16T14:51:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-16T14:51:41,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-16T14:51:41,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9080 sec 2024-11-16T14:51:41,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.9140 sec 2024-11-16T14:51:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T14:51:41,391 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:41,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:51:41,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-16T14:51:41,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-16T14:51:41,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-16T14:51:41,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-16T14:51:41,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-16T14:51:41,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:51:41,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2492 2024-11-16T14:51:41,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2464 2024-11-16T14:51:41,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:51:41,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1128 2024-11-16T14:51:41,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3380 rows 2024-11-16T14:51:41,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1113 2024-11-16T14:51:41,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3338 rows 2024-11-16T14:51:41,392 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:51:41,392 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:51:41,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:41,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:41,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:51:41,396 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:51:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:51:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T14:51:41,402 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768701402"}]},"ts":"1731768701402"} 2024-11-16T14:51:41,404 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:51:41,404 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:51:41,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:51:41,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c410da0e7ea4faea21a8b41f311130b9, UNASSIGN}] 2024-11-16T14:51:41,411 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c410da0e7ea4faea21a8b41f311130b9, UNASSIGN 2024-11-16T14:51:41,412 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=c410da0e7ea4faea21a8b41f311130b9, regionState=CLOSING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:41,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c410da0e7ea4faea21a8b41f311130b9, UNASSIGN because future has completed 2024-11-16T14:51:41,415 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:51:41,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:51:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T14:51:41,571 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:41,572 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:51:41,572 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing c410da0e7ea4faea21a8b41f311130b9, disabling compactions & flushes 2024-11-16T14:51:41,572 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:41,572 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:41,572 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. after waiting 0 ms 2024-11-16T14:51:41,572 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:41,572 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing c410da0e7ea4faea21a8b41f311130b9 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T14:51:41,573 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=A 2024-11-16T14:51:41,573 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:41,573 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=B 2024-11-16T14:51:41,573 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:41,573 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c410da0e7ea4faea21a8b41f311130b9, store=C 2024-11-16T14:51:41,573 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:41,577 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/4f78b8e2cdb24a05990a374350156727 is 50, key is test_row_0/A:col10/1731768700101/Put/seqid=0 2024-11-16T14:51:41,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741942_1118 (size=12301) 2024-11-16T14:51:41,710 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:51:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T14:51:41,982 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/4f78b8e2cdb24a05990a374350156727 2024-11-16T14:51:41,999 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/ba8fbb5a5b734d04b5d32f6883eaf709 is 50, key is test_row_0/B:col10/1731768700101/Put/seqid=0 2024-11-16T14:51:42,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741943_1119 (size=12301) 2024-11-16T14:51:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T14:51:42,404 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/ba8fbb5a5b734d04b5d32f6883eaf709 2024-11-16T14:51:42,411 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/4b9b79e99bb04ec4b15b03b8c92d7b72 is 50, key is test_row_0/C:col10/1731768700101/Put/seqid=0 2024-11-16T14:51:42,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741944_1120 (size=12301) 2024-11-16T14:51:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T14:51:42,816 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/4b9b79e99bb04ec4b15b03b8c92d7b72 2024-11-16T14:51:42,822 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/A/4f78b8e2cdb24a05990a374350156727 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4f78b8e2cdb24a05990a374350156727 2024-11-16T14:51:42,829 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4f78b8e2cdb24a05990a374350156727, entries=150, sequenceid=511, filesize=12.0 K 2024-11-16T14:51:42,831 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/B/ba8fbb5a5b734d04b5d32f6883eaf709 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/ba8fbb5a5b734d04b5d32f6883eaf709 2024-11-16T14:51:42,838 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/ba8fbb5a5b734d04b5d32f6883eaf709, entries=150, sequenceid=511, filesize=12.0 K 2024-11-16T14:51:42,840 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/.tmp/C/4b9b79e99bb04ec4b15b03b8c92d7b72 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4b9b79e99bb04ec4b15b03b8c92d7b72 2024-11-16T14:51:42,844 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4b9b79e99bb04ec4b15b03b8c92d7b72, entries=150, sequenceid=511, filesize=12.0 K 2024-11-16T14:51:42,845 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c410da0e7ea4faea21a8b41f311130b9 in 1273ms, sequenceid=511, compaction requested=true 2024-11-16T14:51:42,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/9f4a21c6272c4d20b7fa084b13348fd0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b34512986b794b4fb352fc865ed073be, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/138192596162420684f9840c18764218, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8f2f1b0edf544ed7908fc717d45d8137, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/bbf5bd14bc64402e83b861a0039836c0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/70b69d3d6b8344779e598a6001330a05, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/056b0cc2618448db854af70f81a83d49, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/6e1a413203b74aa9ae6d68e3b7f0f98f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/09861853e87f4d1b972419eb90ecceb9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f18ef46f94cf40abab026d4ff4088a3c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8ba3284ef1204c32986fee9e9cc17596, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/ed81d72f05c34747a0773f895b42ff92, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/630bb7a11abf4b058d8255bbf8741ead, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b72ddefc95e4477896e0b69a0a679129, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/85da1d14c2524f378bf1ce296c882c31, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4dc5a9579a44487f87dc672e73407ddc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f613da36a79e48598ec534b60b215cd9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/877a9c4bc75c4a099fd320313f1f68c0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4675e01685f548c5a31ff3414dac83a6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3ff2b0c436cf43f48df2d9dc099ce670, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/c2b4b36cc4314d6cba1883ee6ca796a4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b9e249c05c854305af22d737a0be09fa, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8bd0c618600247a0a41d0e6ee65941cd, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3552474408b44ca7b1eb7180bb9d10b1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/d98639f3210a4e009319130f3b467c6e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/79bda35388004d529147d3b9b413de99, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/0b2843da4f4c4fbcb7315d1b6ed5d106, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/186cbe74c77e4c229c4a48d367085f4d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/aa9ddbf701a640d89092fa5949afc652, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/a60f5c2e5df5449ca574b818602444a9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/fd84e0dc646840d18af851df30ce5564, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/60fc4178224a4c13becb7e8252c38bc2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/61548f29988b444bb1f3bda7ae850310] to archive 2024-11-16T14:51:42,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:51:42,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/9f4a21c6272c4d20b7fa084b13348fd0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/9f4a21c6272c4d20b7fa084b13348fd0 2024-11-16T14:51:42,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b34512986b794b4fb352fc865ed073be to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b34512986b794b4fb352fc865ed073be 2024-11-16T14:51:42,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/138192596162420684f9840c18764218 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/138192596162420684f9840c18764218 2024-11-16T14:51:42,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8f2f1b0edf544ed7908fc717d45d8137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8f2f1b0edf544ed7908fc717d45d8137 2024-11-16T14:51:42,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/bbf5bd14bc64402e83b861a0039836c0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/bbf5bd14bc64402e83b861a0039836c0 2024-11-16T14:51:42,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/70b69d3d6b8344779e598a6001330a05 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/70b69d3d6b8344779e598a6001330a05 2024-11-16T14:51:42,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/056b0cc2618448db854af70f81a83d49 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/056b0cc2618448db854af70f81a83d49 2024-11-16T14:51:42,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/6e1a413203b74aa9ae6d68e3b7f0f98f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/6e1a413203b74aa9ae6d68e3b7f0f98f 2024-11-16T14:51:42,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/09861853e87f4d1b972419eb90ecceb9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/09861853e87f4d1b972419eb90ecceb9 2024-11-16T14:51:42,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f18ef46f94cf40abab026d4ff4088a3c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f18ef46f94cf40abab026d4ff4088a3c 2024-11-16T14:51:42,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8ba3284ef1204c32986fee9e9cc17596 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8ba3284ef1204c32986fee9e9cc17596 2024-11-16T14:51:42,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/ed81d72f05c34747a0773f895b42ff92 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/ed81d72f05c34747a0773f895b42ff92 2024-11-16T14:51:42,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/630bb7a11abf4b058d8255bbf8741ead to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/630bb7a11abf4b058d8255bbf8741ead 2024-11-16T14:51:42,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b72ddefc95e4477896e0b69a0a679129 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b72ddefc95e4477896e0b69a0a679129 2024-11-16T14:51:42,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/85da1d14c2524f378bf1ce296c882c31 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/85da1d14c2524f378bf1ce296c882c31 2024-11-16T14:51:42,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4dc5a9579a44487f87dc672e73407ddc to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4dc5a9579a44487f87dc672e73407ddc 2024-11-16T14:51:42,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f613da36a79e48598ec534b60b215cd9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/f613da36a79e48598ec534b60b215cd9 2024-11-16T14:51:42,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/877a9c4bc75c4a099fd320313f1f68c0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/877a9c4bc75c4a099fd320313f1f68c0 2024-11-16T14:51:42,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4675e01685f548c5a31ff3414dac83a6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4675e01685f548c5a31ff3414dac83a6 2024-11-16T14:51:42,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3ff2b0c436cf43f48df2d9dc099ce670 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3ff2b0c436cf43f48df2d9dc099ce670 2024-11-16T14:51:42,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/c2b4b36cc4314d6cba1883ee6ca796a4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/c2b4b36cc4314d6cba1883ee6ca796a4 2024-11-16T14:51:42,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b9e249c05c854305af22d737a0be09fa to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/b9e249c05c854305af22d737a0be09fa 2024-11-16T14:51:42,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8bd0c618600247a0a41d0e6ee65941cd to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/8bd0c618600247a0a41d0e6ee65941cd 2024-11-16T14:51:42,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3552474408b44ca7b1eb7180bb9d10b1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/3552474408b44ca7b1eb7180bb9d10b1 2024-11-16T14:51:42,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/d98639f3210a4e009319130f3b467c6e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/d98639f3210a4e009319130f3b467c6e 2024-11-16T14:51:42,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/79bda35388004d529147d3b9b413de99 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/79bda35388004d529147d3b9b413de99 2024-11-16T14:51:42,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/0b2843da4f4c4fbcb7315d1b6ed5d106 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/0b2843da4f4c4fbcb7315d1b6ed5d106 2024-11-16T14:51:42,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/186cbe74c77e4c229c4a48d367085f4d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/186cbe74c77e4c229c4a48d367085f4d 2024-11-16T14:51:42,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/aa9ddbf701a640d89092fa5949afc652 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/aa9ddbf701a640d89092fa5949afc652 2024-11-16T14:51:42,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/a60f5c2e5df5449ca574b818602444a9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/a60f5c2e5df5449ca574b818602444a9 2024-11-16T14:51:42,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/fd84e0dc646840d18af851df30ce5564 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/fd84e0dc646840d18af851df30ce5564 2024-11-16T14:51:42,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/60fc4178224a4c13becb7e8252c38bc2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/60fc4178224a4c13becb7e8252c38bc2 2024-11-16T14:51:42,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/61548f29988b444bb1f3bda7ae850310 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/61548f29988b444bb1f3bda7ae850310 2024-11-16T14:51:42,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/ff426543b83b4c29bc5d8bbcf2779bde, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/8509cadbced54269bc11ea4eff6aaf24, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9c2746483a55448098070a7727b19bf5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/c526034fa3dc4b90b534346dae7af218, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/62cf354da1ec45cdb9f212895f2fe8ba, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/db390dfacd6b4379bfbaf34ddcf55c46, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/bfb602060920478abba7c879354d07e7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/346119fc0e5040aebc953d963c1beafe, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/15561e60737f484c80177cd36de85c65, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fc2b31a910964548802a4b56c97cd7f4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4537cd17c1c242638b6e409a23b63094, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fba5013667d54b45a581e8a758959e97, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/747aac5f0b2c418fb7fadf11d6d58bf7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/953178a15aa0474fa8bc25f3165dd59b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/715cf51311b04a1dbf736095ddefb1df, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/7cf0835942bf4b749ab4f3faa43e38f6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/b4d6900670974d028ebf741b9b2cbfc1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fe9b621d4c3c4f2c8dec6584c93fd07c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4cabd4ae79774f2289db85722a11156b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f91d2590efff495e986228c61aec3821, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/e14cb691cdcd472f99b460decf91a3cc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/1aaec0fac25f4b768579f97db5fa0d35, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/cc83671443ff4704900a648a98703ace, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/81b40b75d3b54f018dfe71b55749cd40, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/2509c351baf64a1bb30111cc844951a4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9194476557014e99b698ec86b3f44e69, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/42d6c4cd554b47a39dc910a9e8624fcf, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f45c9ae7c5674088b765c79bf05ece0c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fa99cddb32a54500a4b1a79757a79fd6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/d36f5d5a72a84d38b8e5f493ef1fc1e9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/776005a442d2461fa2a93e0c098bde68, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4feb35c9e0a446988014bd28d4b46656, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/835b60ca773e4c7586a75d18c0f6e627] to archive 2024-11-16T14:51:42,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:51:42,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/ff426543b83b4c29bc5d8bbcf2779bde to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/ff426543b83b4c29bc5d8bbcf2779bde 2024-11-16T14:51:42,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/8509cadbced54269bc11ea4eff6aaf24 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/8509cadbced54269bc11ea4eff6aaf24 2024-11-16T14:51:42,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9c2746483a55448098070a7727b19bf5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9c2746483a55448098070a7727b19bf5 2024-11-16T14:51:42,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/c526034fa3dc4b90b534346dae7af218 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/c526034fa3dc4b90b534346dae7af218 2024-11-16T14:51:42,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/62cf354da1ec45cdb9f212895f2fe8ba to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/62cf354da1ec45cdb9f212895f2fe8ba 2024-11-16T14:51:42,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/db390dfacd6b4379bfbaf34ddcf55c46 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/db390dfacd6b4379bfbaf34ddcf55c46 2024-11-16T14:51:42,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/bfb602060920478abba7c879354d07e7 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/bfb602060920478abba7c879354d07e7 2024-11-16T14:51:42,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/346119fc0e5040aebc953d963c1beafe to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/346119fc0e5040aebc953d963c1beafe 2024-11-16T14:51:42,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/15561e60737f484c80177cd36de85c65 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/15561e60737f484c80177cd36de85c65 2024-11-16T14:51:42,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fc2b31a910964548802a4b56c97cd7f4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fc2b31a910964548802a4b56c97cd7f4 2024-11-16T14:51:42,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4537cd17c1c242638b6e409a23b63094 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4537cd17c1c242638b6e409a23b63094 2024-11-16T14:51:42,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fba5013667d54b45a581e8a758959e97 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fba5013667d54b45a581e8a758959e97 2024-11-16T14:51:42,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/747aac5f0b2c418fb7fadf11d6d58bf7 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/747aac5f0b2c418fb7fadf11d6d58bf7 2024-11-16T14:51:42,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/953178a15aa0474fa8bc25f3165dd59b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/953178a15aa0474fa8bc25f3165dd59b 2024-11-16T14:51:42,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/715cf51311b04a1dbf736095ddefb1df to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/715cf51311b04a1dbf736095ddefb1df 2024-11-16T14:51:42,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/7cf0835942bf4b749ab4f3faa43e38f6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/7cf0835942bf4b749ab4f3faa43e38f6 2024-11-16T14:51:42,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/b4d6900670974d028ebf741b9b2cbfc1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/b4d6900670974d028ebf741b9b2cbfc1 2024-11-16T14:51:42,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fe9b621d4c3c4f2c8dec6584c93fd07c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fe9b621d4c3c4f2c8dec6584c93fd07c 2024-11-16T14:51:42,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4cabd4ae79774f2289db85722a11156b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4cabd4ae79774f2289db85722a11156b 2024-11-16T14:51:42,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f91d2590efff495e986228c61aec3821 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f91d2590efff495e986228c61aec3821 2024-11-16T14:51:42,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/e14cb691cdcd472f99b460decf91a3cc to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/e14cb691cdcd472f99b460decf91a3cc 2024-11-16T14:51:42,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/1aaec0fac25f4b768579f97db5fa0d35 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/1aaec0fac25f4b768579f97db5fa0d35 2024-11-16T14:51:42,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/cc83671443ff4704900a648a98703ace to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/cc83671443ff4704900a648a98703ace 2024-11-16T14:51:42,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/81b40b75d3b54f018dfe71b55749cd40 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/81b40b75d3b54f018dfe71b55749cd40 2024-11-16T14:51:42,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/2509c351baf64a1bb30111cc844951a4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/2509c351baf64a1bb30111cc844951a4 2024-11-16T14:51:42,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9194476557014e99b698ec86b3f44e69 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/9194476557014e99b698ec86b3f44e69 2024-11-16T14:51:42,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/42d6c4cd554b47a39dc910a9e8624fcf to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/42d6c4cd554b47a39dc910a9e8624fcf 2024-11-16T14:51:42,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f45c9ae7c5674088b765c79bf05ece0c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f45c9ae7c5674088b765c79bf05ece0c 2024-11-16T14:51:42,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fa99cddb32a54500a4b1a79757a79fd6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/fa99cddb32a54500a4b1a79757a79fd6 2024-11-16T14:51:42,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/d36f5d5a72a84d38b8e5f493ef1fc1e9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/d36f5d5a72a84d38b8e5f493ef1fc1e9 2024-11-16T14:51:42,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/776005a442d2461fa2a93e0c098bde68 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/776005a442d2461fa2a93e0c098bde68 2024-11-16T14:51:42,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4feb35c9e0a446988014bd28d4b46656 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/4feb35c9e0a446988014bd28d4b46656 2024-11-16T14:51:42,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/835b60ca773e4c7586a75d18c0f6e627 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/835b60ca773e4c7586a75d18c0f6e627 2024-11-16T14:51:42,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f4e29bfb3b21416d9099ffac8de0d8ed, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4ce8fe50bb7e4a88a45c9a44e9d3a311, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/5b49a6a9f54440a68617a6a2cf69d3bd, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8942cb50b97d4495902a479ffb78e0c2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/1a589f9edb004af79be2a042200a841e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/5ec535cfcd254c61bb279ccc95b984f8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/83de515188204948ab443e92ff91cee5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/ada52e2fa0024730a79a8cc84eb68fed, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8611ccbf2ca7465284a788e768c13b9e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/9da6659ac17548e59e8a1e69df5181a5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/6a59f9a55e0b4bd3b0e8ad947e82a1f0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/e336bea89474483981f8c9e1e69ad7b9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/681f1ab4559546ec8689172a9d9256f1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f242d70ae8554736bbd9828919928f9c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/a5c78557fd4e41bd8ed66e08e5d3f6a6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0a4592fb2de49a7b5157a86553608b7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/0a68f9d9fd4f409cb6fd80012aa0775a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/7be52233a0924d698ab2d31668eb2efb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/592e33cb967f4a0695eb72311d846fb3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8dbc65569376419495e7088d4be06a40, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d55ac5ec3abf48739811f84533346ae6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/00118992bc294a28bb1bfe6bd29b4ff1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4b63d2c88892498b92e9e004e6c68494, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/c82e61ea0c324529807013260a935aed, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/bfd8bf8596e547fa95f407a860c0c308, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/2b0ea986dfaa499c967ee055152e714b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/beeda26eb1474c13824f72f9206a0387, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/48a6723c34164f8c8ed3b073c74a74fe, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/09fa8a40cef3495fb596534df75c477c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/575cd445787046fb88db840d47ac1973, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d9e4dbe754a64953adc22a19a60b78e5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/73d112d1ddf24903bc6e26d6e4b78143, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0cf51ffb4e74e4cb71becf537ebb4b2] to archive 2024-11-16T14:51:42,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:51:42,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f4e29bfb3b21416d9099ffac8de0d8ed to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f4e29bfb3b21416d9099ffac8de0d8ed 2024-11-16T14:51:42,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4ce8fe50bb7e4a88a45c9a44e9d3a311 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4ce8fe50bb7e4a88a45c9a44e9d3a311 2024-11-16T14:51:42,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/5b49a6a9f54440a68617a6a2cf69d3bd to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/5b49a6a9f54440a68617a6a2cf69d3bd 2024-11-16T14:51:42,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8942cb50b97d4495902a479ffb78e0c2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8942cb50b97d4495902a479ffb78e0c2 2024-11-16T14:51:42,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/1a589f9edb004af79be2a042200a841e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/1a589f9edb004af79be2a042200a841e 2024-11-16T14:51:42,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/5ec535cfcd254c61bb279ccc95b984f8 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/5ec535cfcd254c61bb279ccc95b984f8 2024-11-16T14:51:42,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/83de515188204948ab443e92ff91cee5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/83de515188204948ab443e92ff91cee5 2024-11-16T14:51:42,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/ada52e2fa0024730a79a8cc84eb68fed to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/ada52e2fa0024730a79a8cc84eb68fed 2024-11-16T14:51:42,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8611ccbf2ca7465284a788e768c13b9e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8611ccbf2ca7465284a788e768c13b9e 2024-11-16T14:51:42,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/9da6659ac17548e59e8a1e69df5181a5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/9da6659ac17548e59e8a1e69df5181a5 2024-11-16T14:51:42,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/6a59f9a55e0b4bd3b0e8ad947e82a1f0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/6a59f9a55e0b4bd3b0e8ad947e82a1f0 2024-11-16T14:51:42,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/e336bea89474483981f8c9e1e69ad7b9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/e336bea89474483981f8c9e1e69ad7b9 2024-11-16T14:51:42,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/681f1ab4559546ec8689172a9d9256f1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/681f1ab4559546ec8689172a9d9256f1 2024-11-16T14:51:42,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f242d70ae8554736bbd9828919928f9c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/f242d70ae8554736bbd9828919928f9c 2024-11-16T14:51:42,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/a5c78557fd4e41bd8ed66e08e5d3f6a6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/a5c78557fd4e41bd8ed66e08e5d3f6a6 2024-11-16T14:51:42,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0a4592fb2de49a7b5157a86553608b7 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0a4592fb2de49a7b5157a86553608b7 2024-11-16T14:51:42,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/0a68f9d9fd4f409cb6fd80012aa0775a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/0a68f9d9fd4f409cb6fd80012aa0775a 2024-11-16T14:51:42,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/7be52233a0924d698ab2d31668eb2efb to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/7be52233a0924d698ab2d31668eb2efb 2024-11-16T14:51:42,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/592e33cb967f4a0695eb72311d846fb3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/592e33cb967f4a0695eb72311d846fb3 2024-11-16T14:51:42,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8dbc65569376419495e7088d4be06a40 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/8dbc65569376419495e7088d4be06a40 2024-11-16T14:51:42,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d55ac5ec3abf48739811f84533346ae6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d55ac5ec3abf48739811f84533346ae6 2024-11-16T14:51:42,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/00118992bc294a28bb1bfe6bd29b4ff1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/00118992bc294a28bb1bfe6bd29b4ff1 2024-11-16T14:51:42,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4b63d2c88892498b92e9e004e6c68494 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4b63d2c88892498b92e9e004e6c68494 2024-11-16T14:51:42,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/c82e61ea0c324529807013260a935aed to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/c82e61ea0c324529807013260a935aed 2024-11-16T14:51:42,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/bfd8bf8596e547fa95f407a860c0c308 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/bfd8bf8596e547fa95f407a860c0c308 2024-11-16T14:51:42,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/2b0ea986dfaa499c967ee055152e714b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/2b0ea986dfaa499c967ee055152e714b 2024-11-16T14:51:42,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/beeda26eb1474c13824f72f9206a0387 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/beeda26eb1474c13824f72f9206a0387 2024-11-16T14:51:42,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/48a6723c34164f8c8ed3b073c74a74fe to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/48a6723c34164f8c8ed3b073c74a74fe 2024-11-16T14:51:42,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/09fa8a40cef3495fb596534df75c477c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/09fa8a40cef3495fb596534df75c477c 2024-11-16T14:51:42,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/575cd445787046fb88db840d47ac1973 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/575cd445787046fb88db840d47ac1973 2024-11-16T14:51:42,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d9e4dbe754a64953adc22a19a60b78e5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d9e4dbe754a64953adc22a19a60b78e5 2024-11-16T14:51:42,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/73d112d1ddf24903bc6e26d6e4b78143 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/73d112d1ddf24903bc6e26d6e4b78143 2024-11-16T14:51:42,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0cf51ffb4e74e4cb71becf537ebb4b2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/d0cf51ffb4e74e4cb71becf537ebb4b2 2024-11-16T14:51:42,986 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/recovered.edits/514.seqid, newMaxSeqId=514, maxSeqId=1 2024-11-16T14:51:42,986 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. 2024-11-16T14:51:42,987 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for c410da0e7ea4faea21a8b41f311130b9: Waiting for close lock at 1731768701572Running coprocessor pre-close hooks at 1731768701572Disabling compacts and flushes for region at 1731768701572Disabling writes for close at 1731768701572Obtaining lock to block concurrent updates at 1731768701572Preparing flush snapshotting stores in c410da0e7ea4faea21a8b41f311130b9 at 1731768701572Finished memstore snapshotting TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731768701573 (+1 ms)Flushing stores of TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9. at 1731768701573Flushing c410da0e7ea4faea21a8b41f311130b9/A: creating writer at 1731768701573Flushing c410da0e7ea4faea21a8b41f311130b9/A: appending metadata at 1731768701576 (+3 ms)Flushing c410da0e7ea4faea21a8b41f311130b9/A: closing flushed file at 1731768701576Flushing c410da0e7ea4faea21a8b41f311130b9/B: creating writer at 1731768701993 (+417 ms)Flushing c410da0e7ea4faea21a8b41f311130b9/B: appending metadata at 1731768701998 (+5 ms)Flushing c410da0e7ea4faea21a8b41f311130b9/B: closing flushed file at 1731768701998Flushing c410da0e7ea4faea21a8b41f311130b9/C: creating writer at 1731768702408 (+410 ms)Flushing c410da0e7ea4faea21a8b41f311130b9/C: appending metadata at 1731768702411 (+3 ms)Flushing c410da0e7ea4faea21a8b41f311130b9/C: closing flushed file at 1731768702411Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13527ecd: reopening flushed file at 1731768702821 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f62708d: reopening flushed file at 1731768702830 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c3f1566: reopening flushed file at 1731768702839 (+9 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c410da0e7ea4faea21a8b41f311130b9 in 1273ms, sequenceid=511, compaction requested=true at 1731768702845 (+6 ms)Writing region close event to WAL at 1731768702982 (+137 ms)Running coprocessor post-close hooks at 1731768702986 (+4 ms)Closed at 1731768702986 2024-11-16T14:51:42,989 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:42,989 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=c410da0e7ea4faea21a8b41f311130b9, regionState=CLOSED 2024-11-16T14:51:42,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:51:42,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-16T14:51:42,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure c410da0e7ea4faea21a8b41f311130b9, server=fc3fda29f660,43955,1731768673386 in 1.5770 sec 2024-11-16T14:51:42,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-16T14:51:42,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c410da0e7ea4faea21a8b41f311130b9, UNASSIGN in 1.5840 sec 2024-11-16T14:51:42,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-16T14:51:42,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5900 sec 2024-11-16T14:51:42,999 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768702999"}]},"ts":"1731768702999"} 2024-11-16T14:51:43,001 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:51:43,001 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:51:43,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6050 sec 2024-11-16T14:51:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T14:51:43,562 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:43,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:51:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:51:43,574 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:51:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T14:51:43,575 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:51:43,581 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:43,586 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/recovered.edits] 2024-11-16T14:51:43,590 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4f78b8e2cdb24a05990a374350156727 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/4f78b8e2cdb24a05990a374350156727 2024-11-16T14:51:43,591 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/720a1767485a474787038a47a6f64252 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/720a1767485a474787038a47a6f64252 2024-11-16T14:51:43,593 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/e26750c6f3d94093ad59e3e927b4209c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/A/e26750c6f3d94093ad59e3e927b4209c 2024-11-16T14:51:43,595 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/ba8fbb5a5b734d04b5d32f6883eaf709 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/ba8fbb5a5b734d04b5d32f6883eaf709 2024-11-16T14:51:43,596 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/edb7d2b1bdb7407eb2907b499980f5e4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/edb7d2b1bdb7407eb2907b499980f5e4 2024-11-16T14:51:43,597 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f8394085f4094863a1f358c14d9f3dc1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/B/f8394085f4094863a1f358c14d9f3dc1 2024-11-16T14:51:43,600 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/23fd866e6c7a41bb9273b6f1883b6a3b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/23fd866e6c7a41bb9273b6f1883b6a3b 2024-11-16T14:51:43,601 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4b9b79e99bb04ec4b15b03b8c92d7b72 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/4b9b79e99bb04ec4b15b03b8c92d7b72 2024-11-16T14:51:43,602 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/dd031843e5c74b549b5294589b17f453 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/C/dd031843e5c74b549b5294589b17f453 2024-11-16T14:51:43,605 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/recovered.edits/514.seqid to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9/recovered.edits/514.seqid 2024-11-16T14:51:43,605 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/c410da0e7ea4faea21a8b41f311130b9 2024-11-16T14:51:43,605 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:51:43,610 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:51:43,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-16T14:51:43,620 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:51:43,648 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:51:43,649 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:51:43,650 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:51:43,650 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731768703650"}]},"ts":"9223372036854775807"} 2024-11-16T14:51:43,655 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:51:43,655 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c410da0e7ea4faea21a8b41f311130b9, NAME => 'TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:51:43,655 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:51:43,655 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731768703655"}]},"ts":"9223372036854775807"} 2024-11-16T14:51:43,659 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:51:43,660 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:51:43,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-11-16T14:51:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T14:51:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:51:43,681 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:43,694 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1737428431_22 at /127.0.0.1:43528 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1102405766_22 at /127.0.0.1:57220 [Waiting for operation #336] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1102405766_22 at /127.0.0.1:57350 [Waiting for operation #320] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fc3fda29f660:43955-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=320 (was 231) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4972 (was 5524) 2024-11-16T14:51:43,704 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=320, ProcessCount=11, AvailableMemoryMB=4971 2024-11-16T14:51:43,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:51:43,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:51:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:51:43,748 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:51:43,749 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:43,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-16T14:51:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T14:51:43,750 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:51:43,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741945_1121 (size=1061) 2024-11-16T14:51:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T14:51:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T14:51:44,160 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d 2024-11-16T14:51:44,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741946_1122 (size=53) 2024-11-16T14:51:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T14:51:44,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:51:44,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 72d0f116905856f1457e3e7f2ce74a1a, disabling compactions & flushes 2024-11-16T14:51:44,566 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:44,567 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:44,567 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. after waiting 0 ms 2024-11-16T14:51:44,567 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:44,567 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:44,567 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 72d0f116905856f1457e3e7f2ce74a1a: Waiting for close lock at 1731768704566Disabling compacts and flushes for region at 1731768704566Disabling writes for close at 1731768704567 (+1 ms)Writing region close event to WAL at 1731768704567Closed at 1731768704567 2024-11-16T14:51:44,568 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:51:44,569 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731768704568"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731768704568"}]},"ts":"1731768704568"} 2024-11-16T14:51:44,571 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:51:44,572 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:51:44,572 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768704572"}]},"ts":"1731768704572"} 2024-11-16T14:51:44,574 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:51:44,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, ASSIGN}] 2024-11-16T14:51:44,575 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, ASSIGN 2024-11-16T14:51:44,576 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, ASSIGN; state=OFFLINE, location=fc3fda29f660,43955,1731768673386; forceNewPlan=false, retain=false 2024-11-16T14:51:44,727 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=72d0f116905856f1457e3e7f2ce74a1a, regionState=OPENING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:44,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, ASSIGN because future has completed 2024-11-16T14:51:44,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:51:44,885 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:44,885 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:51:44,886 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,886 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:51:44,886 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,886 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,888 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,889 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:51:44,890 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72d0f116905856f1457e3e7f2ce74a1a columnFamilyName A 2024-11-16T14:51:44,890 DEBUG [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T14:51:44,891 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(327): Store=72d0f116905856f1457e3e7f2ce74a1a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:44,891 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,892 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:51:44,892 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72d0f116905856f1457e3e7f2ce74a1a columnFamilyName B 2024-11-16T14:51:44,892 DEBUG [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:44,893 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(327): Store=72d0f116905856f1457e3e7f2ce74a1a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:44,893 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,894 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:51:44,894 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72d0f116905856f1457e3e7f2ce74a1a columnFamilyName C 2024-11-16T14:51:44,894 DEBUG [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:44,894 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(327): Store=72d0f116905856f1457e3e7f2ce74a1a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:44,895 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:44,895 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,895 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,896 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,897 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,897 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,897 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:51:44,898 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,900 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:51:44,900 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 72d0f116905856f1457e3e7f2ce74a1a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66752833, jitterRate=-0.005305275321006775}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:51:44,900 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:44,901 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 72d0f116905856f1457e3e7f2ce74a1a: Running coprocessor pre-open hook at 1731768704886Writing region info on filesystem at 1731768704886Initializing all the Stores at 1731768704887 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768704887Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768704887Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768704887Cleaning up temporary data from old regions at 1731768704897 (+10 ms)Running coprocessor post-open hooks at 1731768704900 (+3 ms)Region opened successfully at 1731768704901 (+1 ms) 2024-11-16T14:51:44,902 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., pid=34, masterSystemTime=1731768704882 2024-11-16T14:51:44,904 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:44,904 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:44,904 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=72d0f116905856f1457e3e7f2ce74a1a, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:44,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:51:44,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-16T14:51:44,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 in 177 msec 2024-11-16T14:51:44,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-16T14:51:44,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, ASSIGN in 335 msec 2024-11-16T14:51:44,912 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:51:44,913 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768704912"}]},"ts":"1731768704912"} 2024-11-16T14:51:44,914 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:51:44,915 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:51:44,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2090 sec 2024-11-16T14:51:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T14:51:45,901 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:45,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e30321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:45,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:45,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:45,903 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c87e551, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:45,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7033508b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:45,907 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:45,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:45,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:45,911 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:51:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fc3fda29f660,35823,1731768672605 2024-11-16T14:51:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17322130 2024-11-16T14:51:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:51:45,913 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:51:45,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:51:45,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:51:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T14:51:45,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741947_1123 (size=1097) 2024-11-16T14:51:46,338 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T14:51:46,338 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T14:51:46,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:51:46,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, REOPEN/MOVE}] 2024-11-16T14:51:46,350 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, REOPEN/MOVE 2024-11-16T14:51:46,350 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=72d0f116905856f1457e3e7f2ce74a1a, regionState=CLOSING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:46,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, REOPEN/MOVE because future has completed 2024-11-16T14:51:46,353 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:51:46,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:51:46,505 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,506 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:51:46,506 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 72d0f116905856f1457e3e7f2ce74a1a, disabling compactions & flushes 2024-11-16T14:51:46,506 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:46,506 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:46,506 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. after waiting 0 ms 2024-11-16T14:51:46,506 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:46,510 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T14:51:46,511 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:46,511 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 72d0f116905856f1457e3e7f2ce74a1a: Waiting for close lock at 1731768706506Running coprocessor pre-close hooks at 1731768706506Disabling compacts and flushes for region at 1731768706506Disabling writes for close at 1731768706506Writing region close event to WAL at 1731768706506Running coprocessor post-close hooks at 1731768706511 (+5 ms)Closed at 1731768706511 2024-11-16T14:51:46,511 WARN [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 72d0f116905856f1457e3e7f2ce74a1a to self. 2024-11-16T14:51:46,513 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=72d0f116905856f1457e3e7f2ce74a1a, regionState=CLOSED 2024-11-16T14:51:46,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:51:46,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-16T14:51:46,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 in 164 msec 2024-11-16T14:51:46,520 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, REOPEN/MOVE; state=CLOSED, location=fc3fda29f660,43955,1731768673386; forceNewPlan=false, retain=true 2024-11-16T14:51:46,671 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=72d0f116905856f1457e3e7f2ce74a1a, regionState=OPENING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:46,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, REOPEN/MOVE because future has completed 2024-11-16T14:51:46,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:51:46,828 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:46,828 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:51:46,828 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,828 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:51:46,829 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,829 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,831 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,832 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:51:46,837 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72d0f116905856f1457e3e7f2ce74a1a columnFamilyName A 2024-11-16T14:51:46,839 DEBUG [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:46,840 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(327): Store=72d0f116905856f1457e3e7f2ce74a1a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:46,840 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,841 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:51:46,841 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72d0f116905856f1457e3e7f2ce74a1a columnFamilyName B 2024-11-16T14:51:46,841 DEBUG [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:46,842 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(327): Store=72d0f116905856f1457e3e7f2ce74a1a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:46,842 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,842 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:51:46,842 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72d0f116905856f1457e3e7f2ce74a1a columnFamilyName C 2024-11-16T14:51:46,843 DEBUG [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:46,843 INFO [StoreOpener-72d0f116905856f1457e3e7f2ce74a1a-1 {}] regionserver.HStore(327): Store=72d0f116905856f1457e3e7f2ce74a1a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:51:46,843 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:46,843 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,844 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,845 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,846 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,846 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,846 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:51:46,848 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,848 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 72d0f116905856f1457e3e7f2ce74a1a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74498986, jitterRate=0.11012139916419983}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:51:46,848 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,849 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 72d0f116905856f1457e3e7f2ce74a1a: Running coprocessor pre-open hook at 1731768706829Writing region info on filesystem at 1731768706829Initializing all the Stores at 1731768706830 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768706830Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768706830Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768706830Cleaning up temporary data from old regions at 1731768706846 (+16 ms)Running coprocessor post-open hooks at 1731768706848 (+2 ms)Region opened successfully at 1731768706849 (+1 ms) 2024-11-16T14:51:46,850 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., pid=39, masterSystemTime=1731768706825 2024-11-16T14:51:46,852 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:46,852 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:46,853 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=72d0f116905856f1457e3e7f2ce74a1a, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:46,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:51:46,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-16T14:51:46,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 in 182 msec 2024-11-16T14:51:46,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-16T14:51:46,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, REOPEN/MOVE in 508 msec 2024-11-16T14:51:46,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-16T14:51:46,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 519 msec 2024-11-16T14:51:46,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 944 msec 2024-11-16T14:51:46,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-16T14:51:46,871 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:46,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9c3dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:46,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:46,873 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f3dcce2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:46,874 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@668a6924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dde8260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:46,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:46,877 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6c7c3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:46,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e28765, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab5c81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:46,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:46,881 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfb867f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:46,883 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787137cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1a430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:46,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:46,885 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f92314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:46,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:46,887 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b65203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbff363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:46,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:46,890 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:46,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:46,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e7b863b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:46,892 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62988b76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292bddff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:46,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:46,894 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3be7022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:46,896 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:46,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cfc5bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a14e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:46,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:46,898 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa90f02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:46,899 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25d51fe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f17c234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:46,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:46,902 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a4ac1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:46,903 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75003b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197f7df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:51:46,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:51:46,905 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:51:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:51:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:51:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@243a0b16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:51:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:51:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:51:46,907 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:51:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be6ca13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:51:46,909 DEBUG [Thread-632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:46,909 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:46,909 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:46,909 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:46,909 DEBUG [Thread-621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:46,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:46,910 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:46,910 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:46,911 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-16T14:51:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:46,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:46,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:46,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:46,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:46,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:46,913 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:51:46,913 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:51:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:46,915 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:46,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:51:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:51:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:51:46,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:46,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:46,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:46,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:51:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:51:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:51:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:51:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:51:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:51:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:51:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:51:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:51:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:46,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:51:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:51:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:51:46,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:46,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:51:46,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:46,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169fbf79f3d325415a944229cdaf0162fb_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768706926/Put/seqid=0 2024-11-16T14:51:47,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741948_1124 (size=19474) 2024-11-16T14:51:47,012 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:47,020 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169fbf79f3d325415a944229cdaf0162fb_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169fbf79f3d325415a944229cdaf0162fb_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:47,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768706940, max timestamp=1731768707018Num uniques 859; 2024-11-16T14:51:47,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:51:47,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1731768706940, max timestamp=1731768707018Num uniques 859; 2024-11-16T14:51:47,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-16T14:51:47,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768706940, max timestamp=1731768707021Num uniques 859; 2024-11-16T14:51:47,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:51:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:51:47,035 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/a53d9442d0a14285aa4395b7480d660e, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:47,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/a53d9442d0a14285aa4395b7480d660e is 175, key is test_row_0/A:col10/1731768706926/Put/seqid=0 2024-11-16T14:51:47,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741949_1125 (size=56733) 2024-11-16T14:51:47,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T14:51:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:47,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768767066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768767074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768767076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768767080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768767088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768767202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768767203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768767203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768767203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768767213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T14:51:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:47,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:51:47,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T14:51:47,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:47,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:47,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768767421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768767421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768767421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768767421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768767431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,468 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/a53d9442d0a14285aa4395b7480d660e 2024-11-16T14:51:47,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/41dbce106cbd46508273c1c5595a18a3 is 50, key is test_row_0/B:col10/1731768706926/Put/seqid=0 2024-11-16T14:51:47,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T14:51:47,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:47,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:47,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:47,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:51:47,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741950_1126 (size=12001) 2024-11-16T14:51:47,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/41dbce106cbd46508273c1c5595a18a3 2024-11-16T14:51:47,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/c0ef15ec75be44e0b84bc685b8e79b79 is 50, key is test_row_0/C:col10/1731768706926/Put/seqid=0 2024-11-16T14:51:47,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741951_1127 (size=12001) 2024-11-16T14:51:47,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/c0ef15ec75be44e0b84bc685b8e79b79 2024-11-16T14:51:47,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/a53d9442d0a14285aa4395b7480d660e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a53d9442d0a14285aa4395b7480d660e 2024-11-16T14:51:47,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a53d9442d0a14285aa4395b7480d660e, entries=300, sequenceid=14, filesize=55.4 K 2024-11-16T14:51:47,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/41dbce106cbd46508273c1c5595a18a3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/41dbce106cbd46508273c1c5595a18a3 2024-11-16T14:51:47,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/41dbce106cbd46508273c1c5595a18a3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T14:51:47,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/c0ef15ec75be44e0b84bc685b8e79b79 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/c0ef15ec75be44e0b84bc685b8e79b79 2024-11-16T14:51:47,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/c0ef15ec75be44e0b84bc685b8e79b79, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T14:51:47,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=145.36 KB/148850 for 72d0f116905856f1457e3e7f2ce74a1a in 727ms, sequenceid=14, compaction requested=false 2024-11-16T14:51:47,665 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T14:51:47,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:47,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T14:51:47,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:47,696 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=145.36 KB heapSize=401 KB 2024-11-16T14:51:47,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:51:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:51:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:51:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163b08533a7250413d85533b419640657d_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768707074/Put/seqid=0 2024-11-16T14:51:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768767745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768767746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741952_1128 (size=12154) 2024-11-16T14:51:47,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:47,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768767756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768767758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768767758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,775 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163b08533a7250413d85533b419640657d_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163b08533a7250413d85533b419640657d_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:47,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/9d2184aad52d416782824a0b3ed0093a, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:47,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/9d2184aad52d416782824a0b3ed0093a is 175, key is test_row_0/A:col10/1731768707074/Put/seqid=0 2024-11-16T14:51:47,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741953_1129 (size=30955) 2024-11-16T14:51:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768767861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768767861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768767881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768767881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:47,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768767881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:51:48,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768768071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768768071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768768092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768768092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768768100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,210 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/9d2184aad52d416782824a0b3ed0093a 2024-11-16T14:51:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/f1612f41603e44bfaa8e2122f7871489 is 50, key is test_row_0/B:col10/1731768707074/Put/seqid=0 2024-11-16T14:51:48,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741954_1130 (size=12001) 2024-11-16T14:51:48,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768768391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768768392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768768400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768768411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768768411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,551 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:51:48,636 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/f1612f41603e44bfaa8e2122f7871489 2024-11-16T14:51:48,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/4a38dd4a291742e5b017a88c70d4e236 is 50, key is test_row_0/C:col10/1731768707074/Put/seqid=0 2024-11-16T14:51:48,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741955_1131 (size=12001) 2024-11-16T14:51:48,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768768901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768768901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768768921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768768921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:48,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768768921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:49,057 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/4a38dd4a291742e5b017a88c70d4e236 2024-11-16T14:51:49,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/9d2184aad52d416782824a0b3ed0093a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/9d2184aad52d416782824a0b3ed0093a 2024-11-16T14:51:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:51:49,083 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/9d2184aad52d416782824a0b3ed0093a, entries=150, sequenceid=49, filesize=30.2 K 2024-11-16T14:51:49,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/f1612f41603e44bfaa8e2122f7871489 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f1612f41603e44bfaa8e2122f7871489 2024-11-16T14:51:49,095 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f1612f41603e44bfaa8e2122f7871489, entries=150, sequenceid=49, filesize=11.7 K 2024-11-16T14:51:49,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/4a38dd4a291742e5b017a88c70d4e236 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a38dd4a291742e5b017a88c70d4e236 2024-11-16T14:51:49,105 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a38dd4a291742e5b017a88c70d4e236, entries=150, sequenceid=49, filesize=11.7 K 2024-11-16T14:51:49,107 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~400.95 KB/410576, currentSize=46.96 KB/48090 for 72d0f116905856f1457e3e7f2ce74a1a in 1411ms, sequenceid=49, compaction requested=false 2024-11-16T14:51:49,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:49,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:49,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-16T14:51:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-16T14:51:49,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-16T14:51:49,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1960 sec 2024-11-16T14:51:49,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.2060 sec 2024-11-16T14:51:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:49,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:51:49,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:51:49,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:49,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:51:49,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:49,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:51:49,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:49,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f75d84d1a8514a54bb9867871a4e4024_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768709910/Put/seqid=0 2024-11-16T14:51:49,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741956_1132 (size=12154) 2024-11-16T14:51:49,954 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:49,960 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f75d84d1a8514a54bb9867871a4e4024_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f75d84d1a8514a54bb9867871a4e4024_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:49,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/f16a875f0c88466691f65093c07da06a, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:49,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/f16a875f0c88466691f65093c07da06a is 175, key is test_row_0/A:col10/1731768709910/Put/seqid=0 2024-11-16T14:51:49,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741957_1133 (size=30955) 2024-11-16T14:51:49,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768769985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:49,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768769987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:49,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731768709943, max timestamp=1731768709992Num uniques 859; 2024-11-16T14:51:49,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:49,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T14:51:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768769989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768709943, max timestamp=1731768709992Num uniques 859; 2024-11-16T14:51:50,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:51:50,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768709943, max timestamp=1731768709992Num uniques 859; 2024-11-16T14:51:50,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:51:50,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768770076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768770078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768770101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768770101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768770101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768770191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768770191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768770310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768770310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768770310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,369 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/f16a875f0c88466691f65093c07da06a 2024-11-16T14:51:50,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/22972ce4669645f2ac013f2a781f729c is 50, key is test_row_0/B:col10/1731768709910/Put/seqid=0 2024-11-16T14:51:50,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741958_1134 (size=12001) 2024-11-16T14:51:50,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/22972ce4669645f2ac013f2a781f729c 2024-11-16T14:51:50,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768770401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768770402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/fcd547d414cd4edfa2bdf8ebf1f9432f is 50, key is test_row_0/C:col10/1731768709910/Put/seqid=0 2024-11-16T14:51:50,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741959_1135 (size=12001) 2024-11-16T14:51:50,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/fcd547d414cd4edfa2bdf8ebf1f9432f 2024-11-16T14:51:50,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/f16a875f0c88466691f65093c07da06a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/f16a875f0c88466691f65093c07da06a 2024-11-16T14:51:50,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/f16a875f0c88466691f65093c07da06a, entries=150, sequenceid=60, filesize=30.2 K 2024-11-16T14:51:50,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/22972ce4669645f2ac013f2a781f729c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/22972ce4669645f2ac013f2a781f729c 2024-11-16T14:51:50,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/22972ce4669645f2ac013f2a781f729c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-16T14:51:50,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/fcd547d414cd4edfa2bdf8ebf1f9432f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/fcd547d414cd4edfa2bdf8ebf1f9432f 2024-11-16T14:51:50,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/fcd547d414cd4edfa2bdf8ebf1f9432f, entries=150, sequenceid=60, filesize=11.7 K 2024-11-16T14:51:50,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 72d0f116905856f1457e3e7f2ce74a1a in 565ms, sequenceid=60, compaction requested=true 2024-11-16T14:51:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:50,477 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:50,477 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:50,478 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:50,478 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/A is initiating minor compaction (all files) 2024-11-16T14:51:50,478 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:50,478 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/A in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:50,478 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/B is initiating minor compaction (all files) 2024-11-16T14:51:50,479 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/B in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:50,479 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a53d9442d0a14285aa4395b7480d660e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/9d2184aad52d416782824a0b3ed0093a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/f16a875f0c88466691f65093c07da06a] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=115.9 K 2024-11-16T14:51:50,479 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/41dbce106cbd46508273c1c5595a18a3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f1612f41603e44bfaa8e2122f7871489, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/22972ce4669645f2ac013f2a781f729c] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=35.2 K 2024-11-16T14:51:50,479 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:50,479 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a53d9442d0a14285aa4395b7480d660e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/9d2184aad52d416782824a0b3ed0093a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/f16a875f0c88466691f65093c07da06a] 2024-11-16T14:51:50,479 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 41dbce106cbd46508273c1c5595a18a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731768706926 2024-11-16T14:51:50,479 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a53d9442d0a14285aa4395b7480d660e, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731768706918 2024-11-16T14:51:50,480 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f1612f41603e44bfaa8e2122f7871489, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731768707050 2024-11-16T14:51:50,480 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d2184aad52d416782824a0b3ed0093a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731768707050 2024-11-16T14:51:50,480 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 22972ce4669645f2ac013f2a781f729c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731768707740 2024-11-16T14:51:50,481 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f16a875f0c88466691f65093c07da06a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731768707740 2024-11-16T14:51:50,497 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:50,498 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#B#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:50,499 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/ce2ff3d9d6d5492782d18cfc85496afb is 50, key is test_row_0/B:col10/1731768709910/Put/seqid=0 2024-11-16T14:51:50,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741960_1136 (size=12104) 2024-11-16T14:51:50,505 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411168e04030a410144d788ce4b3dbebbda16_72d0f116905856f1457e3e7f2ce74a1a store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:50,510 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411168e04030a410144d788ce4b3dbebbda16_72d0f116905856f1457e3e7f2ce74a1a, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:50,511 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168e04030a410144d788ce4b3dbebbda16_72d0f116905856f1457e3e7f2ce74a1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:50,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741961_1137 (size=4469) 2024-11-16T14:51:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:50,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=138.65 KB heapSize=390.45 KB 2024-11-16T14:51:50,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:51:50,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:50,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:51:50,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:50,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:51:50,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:50,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161cedf014358f47fa967b614e5d32b5f7_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768710057/Put/seqid=0 2024-11-16T14:51:50,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768770642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768770644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768770644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741962_1138 (size=14594) 2024-11-16T14:51:50,656 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:50,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161cedf014358f47fa967b614e5d32b5f7_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161cedf014358f47fa967b614e5d32b5f7_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:50,664 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/16fcaf71259445c68610fb6ae114cbb3, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:50,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/16fcaf71259445c68610fb6ae114cbb3 is 175, key is test_row_0/A:col10/1731768710057/Put/seqid=0 2024-11-16T14:51:50,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741963_1139 (size=39549) 2024-11-16T14:51:50,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768770710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768770720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768770751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768770760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768770760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,912 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/ce2ff3d9d6d5492782d18cfc85496afb as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ce2ff3d9d6d5492782d18cfc85496afb 2024-11-16T14:51:50,920 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/B of 72d0f116905856f1457e3e7f2ce74a1a into ce2ff3d9d6d5492782d18cfc85496afb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:50,920 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:50,920 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/B, priority=13, startTime=1731768710477; duration=0sec 2024-11-16T14:51:50,920 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:50,920 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:B 2024-11-16T14:51:50,920 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:50,924 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:50,924 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/C is initiating minor compaction (all files) 2024-11-16T14:51:50,924 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/C in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:50,924 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/c0ef15ec75be44e0b84bc685b8e79b79, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a38dd4a291742e5b017a88c70d4e236, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/fcd547d414cd4edfa2bdf8ebf1f9432f] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=35.2 K 2024-11-16T14:51:50,925 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting c0ef15ec75be44e0b84bc685b8e79b79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731768706926 2024-11-16T14:51:50,926 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a38dd4a291742e5b017a88c70d4e236, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731768707050 2024-11-16T14:51:50,926 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting fcd547d414cd4edfa2bdf8ebf1f9432f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731768707740 2024-11-16T14:51:50,939 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#A#compaction#118 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:50,940 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/84fc0deb4db344318ccc15f498f0e54a is 175, key is test_row_0/A:col10/1731768709910/Put/seqid=0 2024-11-16T14:51:50,963 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#C#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:50,964 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/b6e5b50c7eb6485981cff3a98f0fc70b is 50, key is test_row_0/C:col10/1731768709910/Put/seqid=0 2024-11-16T14:51:50,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768770961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768770971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:50,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768770971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:50,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741964_1140 (size=31058) 2024-11-16T14:51:50,998 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/84fc0deb4db344318ccc15f498f0e54a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/84fc0deb4db344318ccc15f498f0e54a 2024-11-16T14:51:51,009 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/A of 72d0f116905856f1457e3e7f2ce74a1a into 84fc0deb4db344318ccc15f498f0e54a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:51,009 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:51,009 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/A, priority=13, startTime=1731768710477; duration=0sec 2024-11-16T14:51:51,009 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:51,010 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:A 2024-11-16T14:51:51,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741965_1141 (size=12104) 2024-11-16T14:51:51,020 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/b6e5b50c7eb6485981cff3a98f0fc70b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b6e5b50c7eb6485981cff3a98f0fc70b 2024-11-16T14:51:51,029 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/C of 72d0f116905856f1457e3e7f2ce74a1a into b6e5b50c7eb6485981cff3a98f0fc70b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:51,029 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:51,029 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/C, priority=13, startTime=1731768710477; duration=0sec 2024-11-16T14:51:51,029 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:51,029 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:C 2024-11-16T14:51:51,080 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/16fcaf71259445c68610fb6ae114cbb3 2024-11-16T14:51:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T14:51:51,081 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:51,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-16T14:51:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:51:51,084 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:51,086 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:51,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:51,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/75a55d77da2d467cb106da49dabf0c13 is 50, key is test_row_0/B:col10/1731768710057/Put/seqid=0 2024-11-16T14:51:51,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741966_1142 (size=12001) 2024-11-16T14:51:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:51:51,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768771221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:51,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:51:51,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:51,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:51,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:51,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:51,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768771240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768771281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768771281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768771281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:51,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:51:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:51,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:51:51,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/75a55d77da2d467cb106da49dabf0c13 2024-11-16T14:51:51,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/3e20658ed6a54e4396c6c92adda22b4e is 50, key is test_row_0/C:col10/1731768710057/Put/seqid=0 2024-11-16T14:51:51,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741967_1143 (size=12001) 2024-11-16T14:51:51,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:51:51,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:51,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:51,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:51,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:51:51,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:51,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:51,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:51,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:51:51,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768771790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:51,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768771790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:51,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768771790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:51,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:51:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:51,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:51,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/3e20658ed6a54e4396c6c92adda22b4e 2024-11-16T14:51:51,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/16fcaf71259445c68610fb6ae114cbb3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/16fcaf71259445c68610fb6ae114cbb3 2024-11-16T14:51:51,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/16fcaf71259445c68610fb6ae114cbb3, entries=200, sequenceid=96, filesize=38.6 K 2024-11-16T14:51:51,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/75a55d77da2d467cb106da49dabf0c13 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/75a55d77da2d467cb106da49dabf0c13 2024-11-16T14:51:51,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/75a55d77da2d467cb106da49dabf0c13, entries=150, sequenceid=96, filesize=11.7 K 2024-11-16T14:51:51,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/3e20658ed6a54e4396c6c92adda22b4e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e20658ed6a54e4396c6c92adda22b4e 2024-11-16T14:51:51,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e20658ed6a54e4396c6c92adda22b4e, entries=150, sequenceid=96, filesize=11.7 K 2024-11-16T14:51:51,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~390.41 KB/399776, currentSize=46.96 KB/48090 for 72d0f116905856f1457e3e7f2ce74a1a in 1359ms, sequenceid=96, compaction requested=false 2024-11-16T14:51:51,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:52,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T14:51:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:52,008 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:51:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:51:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:51:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:51:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:52,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160ae78916654c43bd9f8b770c65272269_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768710626/Put/seqid=0 2024-11-16T14:51:52,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741968_1144 (size=12154) 2024-11-16T14:51:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:51:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:52,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:52,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768712272, max timestamp=1731768712294Num uniques 859; 2024-11-16T14:51:52,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:51:52,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731768712272, max timestamp=1731768712294Num uniques 859; 2024-11-16T14:51:52,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-16T14:51:52,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731768712272, max timestamp=1731768712294Num uniques 859; 2024-11-16T14:51:52,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-16T14:51:52,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768772353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:52,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768772358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:52,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:52,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:52,453 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160ae78916654c43bd9f8b770c65272269_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160ae78916654c43bd9f8b770c65272269_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:52,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/3aa14e376b184b61a5f35adf6843033b, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:52,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/3aa14e376b184b61a5f35adf6843033b is 175, key is test_row_0/A:col10/1731768710626/Put/seqid=0 2024-11-16T14:51:52,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768772461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:52,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741969_1145 (size=30955) 2024-11-16T14:51:52,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768772470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768772680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768772680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768772800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768772800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:52,804 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:52,804 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:52,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768772810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:52,820 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:52,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:52,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:52,872 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/3aa14e376b184b61a5f35adf6843033b 2024-11-16T14:51:52,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/baa685051b7f43618f480d801b6e82c3 is 50, key is test_row_0/B:col10/1731768710626/Put/seqid=0 2024-11-16T14:51:52,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741970_1146 (size=12001) 2024-11-16T14:51:53,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768773000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:53,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768773000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:53,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:51:53,293 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/baa685051b7f43618f480d801b6e82c3 2024-11-16T14:51:53,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/d87c505e2d5f4255997981cb3174c352 is 50, key is test_row_0/C:col10/1731768710626/Put/seqid=0 2024-11-16T14:51:53,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741971_1147 (size=12001) 2024-11-16T14:51:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768773510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:53,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768773510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:53,745 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/d87c505e2d5f4255997981cb3174c352 2024-11-16T14:51:53,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/3aa14e376b184b61a5f35adf6843033b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/3aa14e376b184b61a5f35adf6843033b 2024-11-16T14:51:53,753 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/3aa14e376b184b61a5f35adf6843033b, entries=150, sequenceid=109, filesize=30.2 K 2024-11-16T14:51:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/baa685051b7f43618f480d801b6e82c3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/baa685051b7f43618f480d801b6e82c3 2024-11-16T14:51:53,759 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/baa685051b7f43618f480d801b6e82c3, entries=150, sequenceid=109, filesize=11.7 K 2024-11-16T14:51:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/d87c505e2d5f4255997981cb3174c352 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d87c505e2d5f4255997981cb3174c352 2024-11-16T14:51:53,765 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d87c505e2d5f4255997981cb3174c352, entries=150, sequenceid=109, filesize=11.7 K 2024-11-16T14:51:53,766 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.15 KB/134300 for 72d0f116905856f1457e3e7f2ce74a1a in 1758ms, sequenceid=109, compaction requested=true 2024-11-16T14:51:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-16T14:51:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-16T14:51:53,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-16T14:51:53,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6820 sec 2024-11-16T14:51:53,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.6880 sec 2024-11-16T14:51:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:54,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=137.86 KB heapSize=393.31 KB 2024-11-16T14:51:54,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:51:54,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:54,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:51:54,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:54,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:51:54,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:54,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111697d3ae0eef02400984a7e9a3586cdeb6_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768712303/Put/seqid=0 2024-11-16T14:51:54,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741972_1148 (size=14794) 2024-11-16T14:51:54,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768774561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:54,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768774562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:54,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768774671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:54,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768774680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:54,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768774820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:54,826 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:54,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768774831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:54,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768774831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:54,832 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,832 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:54,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768774880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:54,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768774890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:54,965 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:54,970 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111697d3ae0eef02400984a7e9a3586cdeb6_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111697d3ae0eef02400984a7e9a3586cdeb6_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:54,971 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/1056d70e7aea4b93ac081d0572bcce8e, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:54,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/1056d70e7aea4b93ac081d0572bcce8e is 175, key is test_row_0/A:col10/1731768712303/Put/seqid=0 2024-11-16T14:51:54,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741973_1149 (size=39749) 2024-11-16T14:51:55,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768775191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:55,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:55,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:55,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:55,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768775200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T14:51:55,251 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:55,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-16T14:51:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:51:55,254 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:55,254 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:55,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:51:55,376 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/1056d70e7aea4b93ac081d0572bcce8e 2024-11-16T14:51:55,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/3d406d417396483db0a887fb58538850 is 50, key is test_row_0/B:col10/1731768712303/Put/seqid=0 2024-11-16T14:51:55,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741974_1150 (size=12151) 2024-11-16T14:51:55,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/3d406d417396483db0a887fb58538850 2024-11-16T14:51:55,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T14:51:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:55,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:55,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/a5e0670e16f145d68447c78bcacd73c6 is 50, key is test_row_0/C:col10/1731768712303/Put/seqid=0 2024-11-16T14:51:55,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741975_1151 (size=12151) 2024-11-16T14:51:55,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T14:51:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:55,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:55,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:55,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:55,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:51:55,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768775701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:55,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768775710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:55,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T14:51:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:55,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:55,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:55,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/a5e0670e16f145d68447c78bcacd73c6 2024-11-16T14:51:55,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/1056d70e7aea4b93ac081d0572bcce8e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/1056d70e7aea4b93ac081d0572bcce8e 2024-11-16T14:51:55,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/1056d70e7aea4b93ac081d0572bcce8e, entries=200, sequenceid=146, filesize=38.8 K 2024-11-16T14:51:55,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/3d406d417396483db0a887fb58538850 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3d406d417396483db0a887fb58538850 2024-11-16T14:51:55,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3d406d417396483db0a887fb58538850, entries=150, sequenceid=146, filesize=11.9 K 2024-11-16T14:51:55,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/a5e0670e16f145d68447c78bcacd73c6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a5e0670e16f145d68447c78bcacd73c6 2024-11-16T14:51:55,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a5e0670e16f145d68447c78bcacd73c6, entries=150, sequenceid=146, filesize=11.9 K 2024-11-16T14:51:55,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~137.86 KB/141170, heapSize ~393.27 KB/402704, currentSize=40.25 KB/41220 for 72d0f116905856f1457e3e7f2ce74a1a in 1325ms, sequenceid=146, compaction requested=true 2024-11-16T14:51:55,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:55,849 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:55,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:55,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:55,850 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:55,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:55,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:55,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:55,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:55,851 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:55,851 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/A is initiating minor compaction (all files) 2024-11-16T14:51:55,852 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/A in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:55,852 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:55,852 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/B is initiating minor compaction (all files) 2024-11-16T14:51:55,852 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/84fc0deb4db344318ccc15f498f0e54a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/16fcaf71259445c68610fb6ae114cbb3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/3aa14e376b184b61a5f35adf6843033b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/1056d70e7aea4b93ac081d0572bcce8e] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=138.0 K 2024-11-16T14:51:55,852 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/B in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:55,852 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:55,852 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ce2ff3d9d6d5492782d18cfc85496afb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/75a55d77da2d467cb106da49dabf0c13, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/baa685051b7f43618f480d801b6e82c3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3d406d417396483db0a887fb58538850] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=47.1 K 2024-11-16T14:51:55,852 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/84fc0deb4db344318ccc15f498f0e54a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/16fcaf71259445c68610fb6ae114cbb3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/3aa14e376b184b61a5f35adf6843033b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/1056d70e7aea4b93ac081d0572bcce8e] 2024-11-16T14:51:55,858 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84fc0deb4db344318ccc15f498f0e54a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731768707740 2024-11-16T14:51:55,858 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2ff3d9d6d5492782d18cfc85496afb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731768707740 2024-11-16T14:51:55,859 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 75a55d77da2d467cb106da49dabf0c13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731768710057 2024-11-16T14:51:55,859 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16fcaf71259445c68610fb6ae114cbb3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731768710057 2024-11-16T14:51:55,859 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting baa685051b7f43618f480d801b6e82c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731768710626 2024-11-16T14:51:55,859 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa14e376b184b61a5f35adf6843033b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731768710626 2024-11-16T14:51:55,859 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d406d417396483db0a887fb58538850, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731768712303 2024-11-16T14:51:55,860 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1056d70e7aea4b93ac081d0572bcce8e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731768712303 2024-11-16T14:51:55,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T14:51:55,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:55,866 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-16T14:51:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:51:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:51:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:51:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:55,870 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:55,877 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:55,878 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/23e3d75969314f378307f59f8fb9e034 is 50, key is test_row_0/B:col10/1731768712303/Put/seqid=0 2024-11-16T14:51:55,879 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411167e88d886446543a49a2e02da63be620b_72d0f116905856f1457e3e7f2ce74a1a store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:51:55,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741976_1152 (size=12391) 2024-11-16T14:51:55,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116869508d7792d4c7eacfdcdd98d25372c_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768714535/Put/seqid=0 2024-11-16T14:51:55,883 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411167e88d886446543a49a2e02da63be620b_72d0f116905856f1457e3e7f2ce74a1a, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:55,883 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167e88d886446543a49a2e02da63be620b_72d0f116905856f1457e3e7f2ce74a1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:55,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741977_1153 (size=12304) 2024-11-16T14:51:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:55,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741978_1154 (size=4469) 2024-11-16T14:51:55,892 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#A#compaction#129 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:55,893 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/a9a8968a3f6c4c89a0f6d11054e9db93 is 175, key is test_row_0/A:col10/1731768712303/Put/seqid=0 2024-11-16T14:51:55,895 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116869508d7792d4c7eacfdcdd98d25372c_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116869508d7792d4c7eacfdcdd98d25372c_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/4393393e686242c794dd579be0bf20b5, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/4393393e686242c794dd579be0bf20b5 is 175, key is test_row_0/A:col10/1731768714535/Put/seqid=0 2024-11-16T14:51:55,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741979_1155 (size=31345) 2024-11-16T14:51:55,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741980_1156 (size=31105) 2024-11-16T14:51:56,288 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/23e3d75969314f378307f59f8fb9e034 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/23e3d75969314f378307f59f8fb9e034 2024-11-16T14:51:56,294 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/B of 72d0f116905856f1457e3e7f2ce74a1a into 23e3d75969314f378307f59f8fb9e034(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:56,294 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:56,294 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/B, priority=12, startTime=1731768715850; duration=0sec 2024-11-16T14:51:56,294 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:56,294 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:B 2024-11-16T14:51:56,294 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:51:56,296 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:51:56,296 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/C is initiating minor compaction (all files) 2024-11-16T14:51:56,296 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/C in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:56,297 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b6e5b50c7eb6485981cff3a98f0fc70b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e20658ed6a54e4396c6c92adda22b4e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d87c505e2d5f4255997981cb3174c352, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a5e0670e16f145d68447c78bcacd73c6] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=47.1 K 2024-11-16T14:51:56,297 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e5b50c7eb6485981cff3a98f0fc70b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731768707740 2024-11-16T14:51:56,298 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e20658ed6a54e4396c6c92adda22b4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731768710057 2024-11-16T14:51:56,299 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d87c505e2d5f4255997981cb3174c352, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731768710626 2024-11-16T14:51:56,300 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e0670e16f145d68447c78bcacd73c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731768712303 2024-11-16T14:51:56,311 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:56,312 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/7e39b85cb342451bb5434917a5b48ed7 is 50, key is test_row_0/C:col10/1731768712303/Put/seqid=0 2024-11-16T14:51:56,312 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/4393393e686242c794dd579be0bf20b5 2024-11-16T14:51:56,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741981_1157 (size=12391) 2024-11-16T14:51:56,324 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/a9a8968a3f6c4c89a0f6d11054e9db93 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a9a8968a3f6c4c89a0f6d11054e9db93 2024-11-16T14:51:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/09b1587a5907423380632e3e2e402237 is 50, key is test_row_0/B:col10/1731768714535/Put/seqid=0 2024-11-16T14:51:56,332 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/7e39b85cb342451bb5434917a5b48ed7 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/7e39b85cb342451bb5434917a5b48ed7 2024-11-16T14:51:56,337 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/A of 72d0f116905856f1457e3e7f2ce74a1a into a9a8968a3f6c4c89a0f6d11054e9db93(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:56,337 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:56,337 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/A, priority=12, startTime=1731768715849; duration=0sec 2024-11-16T14:51:56,337 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:56,337 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:A 2024-11-16T14:51:56,338 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/C of 72d0f116905856f1457e3e7f2ce74a1a into 7e39b85cb342451bb5434917a5b48ed7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:56,339 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:56,339 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/C, priority=12, startTime=1731768715850; duration=0sec 2024-11-16T14:51:56,339 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:56,339 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:C 2024-11-16T14:51:56,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741982_1158 (size=12151) 2024-11-16T14:51:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:51:56,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:56,755 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/09b1587a5907423380632e3e2e402237 2024-11-16T14:51:56,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/d1f23ad359ce4b26a5f71e7d4ca2c453 is 50, key is test_row_0/C:col10/1731768714535/Put/seqid=0 2024-11-16T14:51:56,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768716736, max timestamp=1731768716770Num uniques 859; 2024-11-16T14:51:56,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768716736, max timestamp=1731768716770Num uniques 859; 2024-11-16T14:51:56,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:51:56,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:51:56,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768716736, max timestamp=1731768716770Num uniques 859; 2024-11-16T14:51:56,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:51:56,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741983_1159 (size=12151) 2024-11-16T14:51:56,789 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/d1f23ad359ce4b26a5f71e7d4ca2c453 2024-11-16T14:51:56,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/4393393e686242c794dd579be0bf20b5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/4393393e686242c794dd579be0bf20b5 2024-11-16T14:51:56,801 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/4393393e686242c794dd579be0bf20b5, entries=150, sequenceid=155, filesize=30.4 K 2024-11-16T14:51:56,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/09b1587a5907423380632e3e2e402237 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/09b1587a5907423380632e3e2e402237 2024-11-16T14:51:56,811 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/09b1587a5907423380632e3e2e402237, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T14:51:56,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/d1f23ad359ce4b26a5f71e7d4ca2c453 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d1f23ad359ce4b26a5f71e7d4ca2c453 2024-11-16T14:51:56,822 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d1f23ad359ce4b26a5f71e7d4ca2c453, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T14:51:56,823 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=118.53 KB/121370 for 72d0f116905856f1457e3e7f2ce74a1a in 957ms, sequenceid=155, compaction requested=false 2024-11-16T14:51:56,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:56,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:56,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-16T14:51:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-16T14:51:56,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-16T14:51:56,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5710 sec 2024-11-16T14:51:56,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5750 sec 2024-11-16T14:51:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:56,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-16T14:51:56,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:51:56,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:56,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:51:56,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:56,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:51:56,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:51:56,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163efb1a8645a04005ba24829ce51a460d_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768716813/Put/seqid=0 2024-11-16T14:51:56,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741984_1160 (size=17284) 2024-11-16T14:51:56,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768776854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:56,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768776863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:56,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:56,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:56,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:56,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768776970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:56,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768776971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:56,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:57,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768777190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:57,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:57,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:57,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:57,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768777190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:57,261 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:57,265 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163efb1a8645a04005ba24829ce51a460d_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163efb1a8645a04005ba24829ce51a460d_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:57,266 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/8d9e8ea064574ab199ab32f41801031e, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:57,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/8d9e8ea064574ab199ab32f41801031e is 175, key is test_row_0/A:col10/1731768716813/Put/seqid=0 2024-11-16T14:51:57,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741985_1161 (size=48389) 2024-11-16T14:51:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T14:51:57,400 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:51:57,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:51:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-16T14:51:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:51:57,403 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:51:57,403 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:51:57,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:51:57,421 INFO [master/fc3fda29f660:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T14:51:57,421 INFO [master/fc3fda29f660:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T14:51:57,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768777501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:57,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768777501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:51:57,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T14:51:57,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:57,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:57,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:57,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:57,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:57,676 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/8d9e8ea064574ab199ab32f41801031e 2024-11-16T14:51:57,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/261174f9a23a413ab3ea8c58d1de36fc is 50, key is test_row_0/B:col10/1731768716813/Put/seqid=0 2024-11-16T14:51:57,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741986_1162 (size=12151) 2024-11-16T14:51:57,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T14:51:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:57,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:51:57,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T14:51:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:57,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768778010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:58,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T14:51:58,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:58,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768778021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:51:58,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/261174f9a23a413ab3ea8c58d1de36fc 2024-11-16T14:51:58,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/b8d76b9240184dcf90d88a8101103969 is 50, key is test_row_0/C:col10/1731768716813/Put/seqid=0 2024-11-16T14:51:58,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741987_1163 (size=12151) 2024-11-16T14:51:58,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T14:51:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T14:51:58,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:58,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T14:51:58,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:58,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:51:58,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/b8d76b9240184dcf90d88a8101103969 2024-11-16T14:51:58,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/8d9e8ea064574ab199ab32f41801031e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8d9e8ea064574ab199ab32f41801031e 2024-11-16T14:51:58,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8d9e8ea064574ab199ab32f41801031e, entries=250, sequenceid=194, filesize=47.3 K 2024-11-16T14:51:58,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/261174f9a23a413ab3ea8c58d1de36fc as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/261174f9a23a413ab3ea8c58d1de36fc 2024-11-16T14:51:58,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/261174f9a23a413ab3ea8c58d1de36fc, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T14:51:58,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/b8d76b9240184dcf90d88a8101103969 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b8d76b9240184dcf90d88a8101103969 2024-11-16T14:51:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:51:58,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b8d76b9240184dcf90d88a8101103969, entries=150, sequenceid=194, filesize=11.9 K 2024-11-16T14:51:58,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for 72d0f116905856f1457e3e7f2ce74a1a in 1719ms, sequenceid=194, compaction requested=true 2024-11-16T14:51:58,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:58,547 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:58,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:51:58,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:58,548 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:58,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:51:58,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:58,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:51:58,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:58,551 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:58,551 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:58,551 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/A is initiating minor compaction (all files) 2024-11-16T14:51:58,551 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/B is initiating minor compaction (all files) 2024-11-16T14:51:58,551 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/B in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,551 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/A in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,551 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a9a8968a3f6c4c89a0f6d11054e9db93, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/4393393e686242c794dd579be0bf20b5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8d9e8ea064574ab199ab32f41801031e] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=108.2 K 2024-11-16T14:51:58,551 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/23e3d75969314f378307f59f8fb9e034, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/09b1587a5907423380632e3e2e402237, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/261174f9a23a413ab3ea8c58d1de36fc] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=35.8 K 2024-11-16T14:51:58,551 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,552 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a9a8968a3f6c4c89a0f6d11054e9db93, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/4393393e686242c794dd579be0bf20b5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8d9e8ea064574ab199ab32f41801031e] 2024-11-16T14:51:58,552 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e3d75969314f378307f59f8fb9e034, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731768712303 2024-11-16T14:51:58,552 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9a8968a3f6c4c89a0f6d11054e9db93, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731768712303 2024-11-16T14:51:58,553 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 09b1587a5907423380632e3e2e402237, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731768714535 2024-11-16T14:51:58,553 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 261174f9a23a413ab3ea8c58d1de36fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731768716813 2024-11-16T14:51:58,554 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4393393e686242c794dd579be0bf20b5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731768714535 2024-11-16T14:51:58,555 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d9e8ea064574ab199ab32f41801031e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731768716802 2024-11-16T14:51:58,562 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:58,563 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/f2579ce41f5b49bb83f25942e4d1db69 is 50, key is test_row_0/B:col10/1731768716813/Put/seqid=0 2024-11-16T14:51:58,565 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:58,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741988_1164 (size=12493) 2024-11-16T14:51:58,578 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/f2579ce41f5b49bb83f25942e4d1db69 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f2579ce41f5b49bb83f25942e4d1db69 2024-11-16T14:51:58,579 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411167cade9ddabc64637b9367f3db6057ee8_72d0f116905856f1457e3e7f2ce74a1a store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:58,586 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411167cade9ddabc64637b9367f3db6057ee8_72d0f116905856f1457e3e7f2ce74a1a, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:58,586 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167cade9ddabc64637b9367f3db6057ee8_72d0f116905856f1457e3e7f2ce74a1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:58,592 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/B of 72d0f116905856f1457e3e7f2ce74a1a into f2579ce41f5b49bb83f25942e4d1db69(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:58,592 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:58,592 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/B, priority=13, startTime=1731768718548; duration=0sec 2024-11-16T14:51:58,592 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:51:58,593 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:B 2024-11-16T14:51:58,593 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:51:58,595 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:51:58,595 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/C is initiating minor compaction (all files) 2024-11-16T14:51:58,595 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/C in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,595 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/7e39b85cb342451bb5434917a5b48ed7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d1f23ad359ce4b26a5f71e7d4ca2c453, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b8d76b9240184dcf90d88a8101103969] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=35.8 K 2024-11-16T14:51:58,596 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e39b85cb342451bb5434917a5b48ed7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731768712303 2024-11-16T14:51:58,596 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d1f23ad359ce4b26a5f71e7d4ca2c453, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731768714535 2024-11-16T14:51:58,597 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting b8d76b9240184dcf90d88a8101103969, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731768716813 2024-11-16T14:51:58,609 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:58,610 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/7d2bd9f951a4457fa6664fc5257f8ea6 is 50, key is test_row_0/C:col10/1731768716813/Put/seqid=0 2024-11-16T14:51:58,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741989_1165 (size=4469) 2024-11-16T14:51:58,615 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#A#compaction#139 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:51:58,616 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/719834477e0246edbcb6487f65bf0be4 is 175, key is test_row_0/A:col10/1731768716813/Put/seqid=0 2024-11-16T14:51:58,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741990_1166 (size=12493) 2024-11-16T14:51:58,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T14:51:58,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:58,629 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:51:58,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:51:58,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:58,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:51:58,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:58,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:51:58,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:51:58,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741991_1167 (size=31447) 2024-11-16T14:51:58,643 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/719834477e0246edbcb6487f65bf0be4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/719834477e0246edbcb6487f65bf0be4 2024-11-16T14:51:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116989256963560421e9d6aed552d1de5ce_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768716852/Put/seqid=0 2024-11-16T14:51:58,656 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/A of 72d0f116905856f1457e3e7f2ce74a1a into 719834477e0246edbcb6487f65bf0be4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:58,656 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:58,656 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/A, priority=13, startTime=1731768718547; duration=0sec 2024-11-16T14:51:58,657 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:58,657 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:A 2024-11-16T14:51:58,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741992_1168 (size=12304) 2024-11-16T14:51:58,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:51:58,675 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116989256963560421e9d6aed552d1de5ce_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116989256963560421e9d6aed552d1de5ce_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/09433ba2bbbe4ef58a88996b5c39db1e, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:51:58,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/09433ba2bbbe4ef58a88996b5c39db1e is 175, key is test_row_0/A:col10/1731768716852/Put/seqid=0 2024-11-16T14:51:58,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741993_1169 (size=31105) 2024-11-16T14:51:58,708 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/09433ba2bbbe4ef58a88996b5c39db1e 2024-11-16T14:51:58,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/3a132061958e42acb1054a6701c542da is 50, key is test_row_0/B:col10/1731768716852/Put/seqid=0 2024-11-16T14:51:58,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741994_1170 (size=12151) 2024-11-16T14:51:58,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:51:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:51:58,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731768718884, max timestamp=1731768718904Num uniques 859; 2024-11-16T14:51:58,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-16T14:51:58,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731768718884, max timestamp=1731768718904Num uniques 859; 2024-11-16T14:51:58,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-16T14:51:58,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731768718884, max timestamp=1731768718904Num uniques 859; 2024-11-16T14:51:58,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-16T14:51:58,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768778936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768778936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768778937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768779020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,022 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,029 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/7d2bd9f951a4457fa6664fc5257f8ea6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/7d2bd9f951a4457fa6664fc5257f8ea6 2024-11-16T14:51:59,034 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/C of 72d0f116905856f1457e3e7f2ce74a1a into 7d2bd9f951a4457fa6664fc5257f8ea6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:51:59,035 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:59,035 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/C, priority=13, startTime=1731768718548; duration=0sec 2024-11-16T14:51:59,035 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:51:59,035 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:C 2024-11-16T14:51:59,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768779030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,038 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:51:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768779051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768779051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768779051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,137 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/3a132061958e42acb1054a6701c542da 2024-11-16T14:51:59,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/cc916b303241405ebdd2a33b7a85472e is 50, key is test_row_0/C:col10/1731768716852/Put/seqid=0 2024-11-16T14:51:59,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741995_1171 (size=12151) 2024-11-16T14:51:59,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768779261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768779261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768779261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,549 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/cc916b303241405ebdd2a33b7a85472e 2024-11-16T14:51:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:51:59,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/09433ba2bbbe4ef58a88996b5c39db1e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/09433ba2bbbe4ef58a88996b5c39db1e 2024-11-16T14:51:59,559 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/09433ba2bbbe4ef58a88996b5c39db1e, entries=150, sequenceid=205, filesize=30.4 K 2024-11-16T14:51:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/3a132061958e42acb1054a6701c542da as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3a132061958e42acb1054a6701c542da 2024-11-16T14:51:59,564 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3a132061958e42acb1054a6701c542da, entries=150, sequenceid=205, filesize=11.9 K 2024-11-16T14:51:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/cc916b303241405ebdd2a33b7a85472e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/cc916b303241405ebdd2a33b7a85472e 2024-11-16T14:51:59,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768779571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768779571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,572 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/cc916b303241405ebdd2a33b7a85472e, entries=150, sequenceid=205, filesize=11.9 K 2024-11-16T14:51:59,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:51:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768779571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:51:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:51:59,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:51:59,573 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 72d0f116905856f1457e3e7f2ce74a1a in 944ms, sequenceid=205, compaction requested=false 2024-11-16T14:51:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:51:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:51:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-16T14:51:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-16T14:51:59,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-16T14:51:59,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1720 sec 2024-11-16T14:51:59,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1790 sec 2024-11-16T14:52:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:00,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=154.31 KB heapSize=421.31 KB 2024-11-16T14:52:00,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:52:00,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:00,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:52:00,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:00,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:52:00,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:00,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111602ed727afc75408687099881868ff0bc_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768718909/Put/seqid=0 2024-11-16T14:52:00,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768780089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768780089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768780093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741996_1172 (size=17284) 2024-11-16T14:52:00,107 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:00,112 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111602ed727afc75408687099881868ff0bc_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111602ed727afc75408687099881868ff0bc_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:00,113 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/71d137da0b8a4f47aad76955eec4df02, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:00,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/71d137da0b8a4f47aad76955eec4df02 is 175, key is test_row_0/A:col10/1731768718909/Put/seqid=0 2024-11-16T14:52:00,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741997_1173 (size=48389) 2024-11-16T14:52:00,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768780211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768780211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768780211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768780420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768780421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768780421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,518 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/71d137da0b8a4f47aad76955eec4df02 2024-11-16T14:52:00,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/f858442f4aa14f918aff61101f4f9abb is 50, key is test_row_0/B:col10/1731768718909/Put/seqid=0 2024-11-16T14:52:00,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741998_1174 (size=12151) 2024-11-16T14:52:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768780730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768780740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768780741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:00,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:00,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/f858442f4aa14f918aff61101f4f9abb 2024-11-16T14:52:00,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/d6658c9d250440ef886f846e9b493744 is 50, key is test_row_0/C:col10/1731768718909/Put/seqid=0 2024-11-16T14:52:00,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741999_1175 (size=12151) 2024-11-16T14:52:01,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768781031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:01,033 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:01,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768781050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:01,053 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:01,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768781251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768781251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768781251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:01,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/d6658c9d250440ef886f846e9b493744 2024-11-16T14:52:01,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/71d137da0b8a4f47aad76955eec4df02 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/71d137da0b8a4f47aad76955eec4df02 2024-11-16T14:52:01,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/71d137da0b8a4f47aad76955eec4df02, entries=250, sequenceid=243, filesize=47.3 K 2024-11-16T14:52:01,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/f858442f4aa14f918aff61101f4f9abb as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f858442f4aa14f918aff61101f4f9abb 2024-11-16T14:52:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f858442f4aa14f918aff61101f4f9abb, entries=150, sequenceid=243, filesize=11.9 K 2024-11-16T14:52:01,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/d6658c9d250440ef886f846e9b493744 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d6658c9d250440ef886f846e9b493744 2024-11-16T14:52:01,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d6658c9d250440ef886f846e9b493744, entries=150, sequenceid=243, filesize=11.9 K 2024-11-16T14:52:01,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~441.19 KB/451776, currentSize=26.84 KB/27480 for 72d0f116905856f1457e3e7f2ce74a1a in 1285ms, sequenceid=243, compaction requested=true 2024-11-16T14:52:01,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:01,371 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:01,372 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:01,373 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:01,373 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/A is initiating minor compaction (all files) 2024-11-16T14:52:01,373 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/A in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:01,373 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:01,373 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/719834477e0246edbcb6487f65bf0be4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/09433ba2bbbe4ef58a88996b5c39db1e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/71d137da0b8a4f47aad76955eec4df02] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=108.3 K 2024-11-16T14:52:01,373 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/B is initiating minor compaction (all files) 2024-11-16T14:52:01,373 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:01,373 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/B in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:01,373 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/719834477e0246edbcb6487f65bf0be4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/09433ba2bbbe4ef58a88996b5c39db1e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/71d137da0b8a4f47aad76955eec4df02] 2024-11-16T14:52:01,373 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f2579ce41f5b49bb83f25942e4d1db69, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3a132061958e42acb1054a6701c542da, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f858442f4aa14f918aff61101f4f9abb] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=35.9 K 2024-11-16T14:52:01,374 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f2579ce41f5b49bb83f25942e4d1db69, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731768716813 2024-11-16T14:52:01,374 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 719834477e0246edbcb6487f65bf0be4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731768716813 2024-11-16T14:52:01,374 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a132061958e42acb1054a6701c542da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731768716845 2024-11-16T14:52:01,374 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09433ba2bbbe4ef58a88996b5c39db1e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731768716845 2024-11-16T14:52:01,375 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71d137da0b8a4f47aad76955eec4df02, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731768718909 2024-11-16T14:52:01,375 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f858442f4aa14f918aff61101f4f9abb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731768718909 2024-11-16T14:52:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,403 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#B#compaction#147 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:01,403 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/0498f020c7924a0b9a1f710e2c4a2541 is 50, key is test_row_0/B:col10/1731768718909/Put/seqid=0 2024-11-16T14:52:01,405 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,415 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116c37c0995b1254950981f40df04822914_72d0f116905856f1457e3e7f2ce74a1a store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,419 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116c37c0995b1254950981f40df04822914_72d0f116905856f1457e3e7f2ce74a1a, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:01,419 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c37c0995b1254950981f40df04822914_72d0f116905856f1457e3e7f2ce74a1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742000_1176 (size=12595) 2024-11-16T14:52:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742001_1177 (size=4469) 2024-11-16T14:52:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T14:52:01,571 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:01,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-16T14:52:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,575 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:52:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,577 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:01,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:52:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T14:52:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,739 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-16T14:52:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:52:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:52:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:52:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116686242da85d842e5a924eb8f7fb33693_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768720091/Put/seqid=0 2024-11-16T14:52:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742002_1178 (size=9814) 2024-11-16T14:52:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,805 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116686242da85d842e5a924eb8f7fb33693_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116686242da85d842e5a924eb8f7fb33693_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/0779be35e26046cc83f1259af4f121e2, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/0779be35e26046cc83f1259af4f121e2 is 175, key is test_row_0/A:col10/1731768720091/Put/seqid=0 2024-11-16T14:52:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742003_1179 (size=22461) 2024-11-16T14:52:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,847 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/0498f020c7924a0b9a1f710e2c4a2541 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/0498f020c7924a0b9a1f710e2c4a2541 2024-11-16T14:52:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,853 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/B of 72d0f116905856f1457e3e7f2ce74a1a into 0498f020c7924a0b9a1f710e2c4a2541(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:01,853 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:01,853 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/B, priority=13, startTime=1731768721372; duration=0sec 2024-11-16T14:52:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,853 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:01,853 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:B 2024-11-16T14:52:01,853 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,856 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,856 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/C is initiating minor compaction (all files) 2024-11-16T14:52:01,856 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/C in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:01,856 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/7d2bd9f951a4457fa6664fc5257f8ea6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/cc916b303241405ebdd2a33b7a85472e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d6658c9d250440ef886f846e9b493744] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=35.9 K 2024-11-16T14:52:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,857 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d2bd9f951a4457fa6664fc5257f8ea6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731768716813 2024-11-16T14:52:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,857 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting cc916b303241405ebdd2a33b7a85472e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731768716845 2024-11-16T14:52:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,859 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#A#compaction#148 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:01,859 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d6658c9d250440ef886f846e9b493744, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731768718909 2024-11-16T14:52:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,859 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/c2a53b624bd54e51ab85a4246bc2467e is 175, key is test_row_0/A:col10/1731768718909/Put/seqid=0 2024-11-16T14:52:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,885 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,885 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/a0ccd12cd34d485b8a862d4ba24ebe30 is 50, key is test_row_0/C:col10/1731768718909/Put/seqid=0 2024-11-16T14:52:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:52:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742004_1180 (size=31549) 2024-11-16T14:52:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742005_1181 (size=12595) 2024-11-16T14:52:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,951 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/a0ccd12cd34d485b8a862d4ba24ebe30 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a0ccd12cd34d485b8a862d4ba24ebe30 2024-11-16T14:52:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,978 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/C of 72d0f116905856f1457e3e7f2ce74a1a into a0ccd12cd34d485b8a862d4ba24ebe30(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:01,978 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:01,978 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/C, priority=13, startTime=1731768721373; duration=0sec 2024-11-16T14:52:01,978 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:01,978 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:C 2024-11-16T14:52:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:52:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,213 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/0779be35e26046cc83f1259af4f121e2 2024-11-16T14:52:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/b90f3988b3e946a890df09c66e786c2a is 50, key is test_row_0/B:col10/1731768720091/Put/seqid=0 2024-11-16T14:52:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742006_1182 (size=9757) 2024-11-16T14:52:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/b90f3988b3e946a890df09c66e786c2a 2024-11-16T14:52:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/3e8955395bee47bda56522151531f07c is 50, key is test_row_0/C:col10/1731768720091/Put/seqid=0 2024-11-16T14:52:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742007_1183 (size=9757) 2024-11-16T14:52:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,317 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/3e8955395bee47bda56522151531f07c 2024-11-16T14:52:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,328 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/c2a53b624bd54e51ab85a4246bc2467e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/c2a53b624bd54e51ab85a4246bc2467e 2024-11-16T14:52:02,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/0779be35e26046cc83f1259af4f121e2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/0779be35e26046cc83f1259af4f121e2 2024-11-16T14:52:02,336 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/0779be35e26046cc83f1259af4f121e2, entries=100, sequenceid=250, filesize=21.9 K 2024-11-16T14:52:02,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/b90f3988b3e946a890df09c66e786c2a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/b90f3988b3e946a890df09c66e786c2a 2024-11-16T14:52:02,342 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/A of 72d0f116905856f1457e3e7f2ce74a1a into c2a53b624bd54e51ab85a4246bc2467e(size=30.8 K), total size for store is 52.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:02,342 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:02,342 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/A, priority=13, startTime=1731768721371; duration=0sec 2024-11-16T14:52:02,343 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:02,343 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:A 2024-11-16T14:52:02,344 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/b90f3988b3e946a890df09c66e786c2a, entries=100, sequenceid=250, filesize=9.5 K 2024-11-16T14:52:02,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/3e8955395bee47bda56522151531f07c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e8955395bee47bda56522151531f07c 2024-11-16T14:52:02,351 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e8955395bee47bda56522151531f07c, entries=100, sequenceid=250, filesize=9.5 K 2024-11-16T14:52:02,352 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=107.34 KB/109920 for 72d0f116905856f1457e3e7f2ce74a1a in 613ms, sequenceid=250, compaction requested=false 2024-11-16T14:52:02,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:02,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:02,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-16T14:52:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-16T14:52:02,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:52:02,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:52:02,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:02,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:52:02,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:02,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:52:02,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:02,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-16T14:52:02,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 777 msec 2024-11-16T14:52:02,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 790 msec 2024-11-16T14:52:02,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165ec9078948ba4dbcaa65eb2346dd8c8d_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768722328/Put/seqid=0 2024-11-16T14:52:02,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742008_1184 (size=12454) 2024-11-16T14:52:02,384 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:02,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768782381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,389 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165ec9078948ba4dbcaa65eb2346dd8c8d_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165ec9078948ba4dbcaa65eb2346dd8c8d_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:02,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768782384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768782384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,392 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/d349fca05d864da9a46328d25dc4462b, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:02,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/d349fca05d864da9a46328d25dc4462b is 175, key is test_row_0/A:col10/1731768722328/Put/seqid=0 2024-11-16T14:52:02,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742009_1185 (size=31255) 2024-11-16T14:52:02,427 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/d349fca05d864da9a46328d25dc4462b 2024-11-16T14:52:02,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/627f16852acb48d3bd752db5ee181758 is 50, key is test_row_0/B:col10/1731768722328/Put/seqid=0 2024-11-16T14:52:02,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742010_1186 (size=12301) 2024-11-16T14:52:02,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/627f16852acb48d3bd752db5ee181758 2024-11-16T14:52:02,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/0bdcbdbee9c84fa4a9fe4bd7d5d0cb1e is 50, key is test_row_0/C:col10/1731768722328/Put/seqid=0 2024-11-16T14:52:02,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768782491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742011_1187 (size=12301) 2024-11-16T14:52:02,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/0bdcbdbee9c84fa4a9fe4bd7d5d0cb1e 2024-11-16T14:52:02,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768782500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768782501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/d349fca05d864da9a46328d25dc4462b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/d349fca05d864da9a46328d25dc4462b 2024-11-16T14:52:02,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/d349fca05d864da9a46328d25dc4462b, entries=150, sequenceid=273, filesize=30.5 K 2024-11-16T14:52:02,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/627f16852acb48d3bd752db5ee181758 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/627f16852acb48d3bd752db5ee181758 2024-11-16T14:52:02,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/627f16852acb48d3bd752db5ee181758, entries=150, sequenceid=273, filesize=12.0 K 2024-11-16T14:52:02,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/0bdcbdbee9c84fa4a9fe4bd7d5d0cb1e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/0bdcbdbee9c84fa4a9fe4bd7d5d0cb1e 2024-11-16T14:52:02,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/0bdcbdbee9c84fa4a9fe4bd7d5d0cb1e, entries=150, sequenceid=273, filesize=12.0 K 2024-11-16T14:52:02,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 72d0f116905856f1457e3e7f2ce74a1a in 169ms, sequenceid=273, compaction requested=true 2024-11-16T14:52:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:02,524 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:02,524 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:02,525 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:02,525 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/A is initiating minor compaction (all files) 2024-11-16T14:52:02,526 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/A in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:02,526 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/c2a53b624bd54e51ab85a4246bc2467e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/0779be35e26046cc83f1259af4f121e2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/d349fca05d864da9a46328d25dc4462b] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=83.3 K 2024-11-16T14:52:02,526 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:02,526 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/c2a53b624bd54e51ab85a4246bc2467e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/0779be35e26046cc83f1259af4f121e2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/d349fca05d864da9a46328d25dc4462b] 2024-11-16T14:52:02,526 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:02,526 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/B is initiating minor compaction (all files) 2024-11-16T14:52:02,526 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/B in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:02,526 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/0498f020c7924a0b9a1f710e2c4a2541, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/b90f3988b3e946a890df09c66e786c2a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/627f16852acb48d3bd752db5ee181758] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=33.8 K 2024-11-16T14:52:02,527 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0498f020c7924a0b9a1f710e2c4a2541, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731768718909 2024-11-16T14:52:02,527 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a53b624bd54e51ab85a4246bc2467e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731768718909 2024-11-16T14:52:02,527 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting b90f3988b3e946a890df09c66e786c2a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731768720087 2024-11-16T14:52:02,527 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0779be35e26046cc83f1259af4f121e2, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731768720087 2024-11-16T14:52:02,527 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d349fca05d864da9a46328d25dc4462b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731768722328 2024-11-16T14:52:02,527 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 627f16852acb48d3bd752db5ee181758, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731768722328 2024-11-16T14:52:02,535 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:02,540 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116d04192d6ceed48bc946bd9097b22b8b8_72d0f116905856f1457e3e7f2ce74a1a store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:02,541 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:02,542 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116d04192d6ceed48bc946bd9097b22b8b8_72d0f116905856f1457e3e7f2ce74a1a, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:02,542 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d04192d6ceed48bc946bd9097b22b8b8_72d0f116905856f1457e3e7f2ce74a1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:02,543 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/d1e60a907f6c409da2860f93f5f471b8 is 50, key is test_row_0/B:col10/1731768722328/Put/seqid=0 2024-11-16T14:52:02,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742012_1188 (size=4469) 2024-11-16T14:52:02,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742013_1189 (size=12847) 2024-11-16T14:52:02,587 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#A#compaction#156 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:02,588 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/fa01bc469b5c4a068a3600d499a0594d is 175, key is test_row_0/A:col10/1731768722328/Put/seqid=0 2024-11-16T14:52:02,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742014_1190 (size=31801) 2024-11-16T14:52:02,621 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/fa01bc469b5c4a068a3600d499a0594d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/fa01bc469b5c4a068a3600d499a0594d 2024-11-16T14:52:02,629 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/A of 72d0f116905856f1457e3e7f2ce74a1a into fa01bc469b5c4a068a3600d499a0594d(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:02,630 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:02,630 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/A, priority=13, startTime=1731768722524; duration=0sec 2024-11-16T14:52:02,630 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:02,630 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:A 2024-11-16T14:52:02,630 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:02,632 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:02,632 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/C is initiating minor compaction (all files) 2024-11-16T14:52:02,632 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/C in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:02,634 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a0ccd12cd34d485b8a862d4ba24ebe30, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e8955395bee47bda56522151531f07c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/0bdcbdbee9c84fa4a9fe4bd7d5d0cb1e] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=33.8 K 2024-11-16T14:52:02,635 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ccd12cd34d485b8a862d4ba24ebe30, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731768718909 2024-11-16T14:52:02,635 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e8955395bee47bda56522151531f07c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731768720087 2024-11-16T14:52:02,636 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bdcbdbee9c84fa4a9fe4bd7d5d0cb1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731768722328 2024-11-16T14:52:02,653 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:02,653 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/4a95f6d3142d4e1195e84f2e5e375d71 is 50, key is test_row_0/C:col10/1731768722328/Put/seqid=0 2024-11-16T14:52:02,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742015_1191 (size=12847) 2024-11-16T14:52:02,684 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/4a95f6d3142d4e1195e84f2e5e375d71 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a95f6d3142d4e1195e84f2e5e375d71 2024-11-16T14:52:02,691 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/C of 72d0f116905856f1457e3e7f2ce74a1a into 4a95f6d3142d4e1195e84f2e5e375d71(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:02,691 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:02,691 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/C, priority=13, startTime=1731768722524; duration=0sec 2024-11-16T14:52:02,691 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:02,691 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:C 2024-11-16T14:52:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T14:52:02,711 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:02,711 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-16T14:52:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:52:02,713 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:02,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:52:02,714 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:02,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:52:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:52:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:52:02,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:02,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c1febb47ce1741698969931f0304d7c6_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768722379/Put/seqid=0 2024-11-16T14:52:02,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742016_1192 (size=14994) 2024-11-16T14:52:02,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768782737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768782738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768782739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:52:02,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768782850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768782850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768782850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:02,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:02,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:02,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:02,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:02,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:02,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:02,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:02,990 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/d1e60a907f6c409da2860f93f5f471b8 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/d1e60a907f6c409da2860f93f5f471b8 2024-11-16T14:52:02,995 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/B of 72d0f116905856f1457e3e7f2ce74a1a into d1e60a907f6c409da2860f93f5f471b8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:02,995 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:02,995 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/B, priority=13, startTime=1731768722524; duration=0sec 2024-11-16T14:52:02,995 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:02,995 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:B 2024-11-16T14:52:03,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:52:03,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768783060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:03,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768783060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:03,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768783060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:03,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:03,137 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:03,140 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c1febb47ce1741698969931f0304d7c6_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c1febb47ce1741698969931f0304d7c6_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:03,142 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/7a7ee45d2e6149cf8a1f45ea54476ce2, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:03,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/7a7ee45d2e6149cf8a1f45ea54476ce2 is 175, key is test_row_0/A:col10/1731768722379/Put/seqid=0 2024-11-16T14:52:03,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742017_1193 (size=39949) 2024-11-16T14:52:03,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:03,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:03,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:03,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:52:03,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768783370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:03,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768783370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:03,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768783370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:03,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:03,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:03,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,548 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/7a7ee45d2e6149cf8a1f45ea54476ce2 2024-11-16T14:52:03,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/ef2ec79b5dde4ddcb6457881a4fe832c is 50, key is test_row_0/B:col10/1731768722379/Put/seqid=0 2024-11-16T14:52:03,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742018_1194 (size=12301) 2024-11-16T14:52:03,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:03,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:03,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:03,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:03,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:52:03,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768783880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768783880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:03,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768783880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:03,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:03,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:03,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:03,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/ef2ec79b5dde4ddcb6457881a4fe832c 2024-11-16T14:52:03,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/8d50b4b2e7724f3a807868d59cce8460 is 50, key is test_row_0/C:col10/1731768722379/Put/seqid=0 2024-11-16T14:52:03,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742019_1195 (size=12301) 2024-11-16T14:52:04,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:04,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:04,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:04,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:04,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:04,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:04,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:04,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:04,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/8d50b4b2e7724f3a807868d59cce8460 2024-11-16T14:52:04,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/7a7ee45d2e6149cf8a1f45ea54476ce2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/7a7ee45d2e6149cf8a1f45ea54476ce2 2024-11-16T14:52:04,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/7a7ee45d2e6149cf8a1f45ea54476ce2, entries=200, sequenceid=289, filesize=39.0 K 2024-11-16T14:52:04,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/ef2ec79b5dde4ddcb6457881a4fe832c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ef2ec79b5dde4ddcb6457881a4fe832c 2024-11-16T14:52:04,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ef2ec79b5dde4ddcb6457881a4fe832c, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T14:52:04,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/8d50b4b2e7724f3a807868d59cce8460 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/8d50b4b2e7724f3a807868d59cce8460 2024-11-16T14:52:04,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/8d50b4b2e7724f3a807868d59cce8460, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T14:52:04,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:04,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:04,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:04,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:04,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:04,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:04,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 72d0f116905856f1457e3e7f2ce74a1a in 1683ms, sequenceid=289, compaction requested=false 2024-11-16T14:52:04,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:04,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T14:52:04,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:04,549 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T14:52:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:52:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:52:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:52:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:04,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ea9c6803b7ef468fba399d69dca7459a_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768722737/Put/seqid=0 2024-11-16T14:52:04,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742020_1196 (size=12454) 2024-11-16T14:52:04,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:52:04,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:04,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:04,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768784920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:04,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:04,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768784927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:04,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768784927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:04,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:04,968 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ea9c6803b7ef468fba399d69dca7459a_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ea9c6803b7ef468fba399d69dca7459a_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:04,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/622e78c644e44c408de1139891b158ed, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/622e78c644e44c408de1139891b158ed is 175, key is test_row_0/A:col10/1731768722737/Put/seqid=0 2024-11-16T14:52:04,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742021_1197 (size=31255) 2024-11-16T14:52:05,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768785030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768785030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768785040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53850 deadline: 1731768785051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,054 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53882 deadline: 1731768785070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,074 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 72d0f116905856f1457e3e7f2ce74a1a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768785240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768785240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768785251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,374 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/622e78c644e44c408de1139891b158ed 2024-11-16T14:52:05,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/ac6094a81bf347da8934fd263bd83e26 is 50, key is test_row_0/B:col10/1731768722737/Put/seqid=0 2024-11-16T14:52:05,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742022_1198 (size=12301) 2024-11-16T14:52:05,387 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/ac6094a81bf347da8934fd263bd83e26 2024-11-16T14:52:05,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/1c761bab9c34408e82e3371a8ffb2a49 is 50, key is test_row_0/C:col10/1731768722737/Put/seqid=0 2024-11-16T14:52:05,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742023_1199 (size=12301) 2024-11-16T14:52:05,418 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/1c761bab9c34408e82e3371a8ffb2a49 2024-11-16T14:52:05,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/622e78c644e44c408de1139891b158ed as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/622e78c644e44c408de1139891b158ed 2024-11-16T14:52:05,428 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/622e78c644e44c408de1139891b158ed, entries=150, sequenceid=308, filesize=30.5 K 2024-11-16T14:52:05,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/ac6094a81bf347da8934fd263bd83e26 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ac6094a81bf347da8934fd263bd83e26 2024-11-16T14:52:05,433 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ac6094a81bf347da8934fd263bd83e26, entries=150, sequenceid=308, filesize=12.0 K 2024-11-16T14:52:05,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/1c761bab9c34408e82e3371a8ffb2a49 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/1c761bab9c34408e82e3371a8ffb2a49 2024-11-16T14:52:05,438 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/1c761bab9c34408e82e3371a8ffb2a49, entries=150, sequenceid=308, filesize=12.0 K 2024-11-16T14:52:05,438 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 72d0f116905856f1457e3e7f2ce74a1a in 889ms, sequenceid=308, compaction requested=true 2024-11-16T14:52:05,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:05,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:05,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-16T14:52:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-16T14:52:05,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-16T14:52:05,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7260 sec 2024-11-16T14:52:05,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.7310 sec 2024-11-16T14:52:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:05,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T14:52:05,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:52:05,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:05,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:52:05,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:05,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:52:05,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:05,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b58e9d5822db44e19a2babe411c4b9b3_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768725554/Put/seqid=0 2024-11-16T14:52:05,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742024_1200 (size=17534) 2024-11-16T14:52:05,573 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:05,578 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b58e9d5822db44e19a2babe411c4b9b3_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b58e9d5822db44e19a2babe411c4b9b3_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:05,579 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/74fa9c5061494727853f1e2e19b5a2ed, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:05,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/74fa9c5061494727853f1e2e19b5a2ed is 175, key is test_row_0/A:col10/1731768725554/Put/seqid=0 2024-11-16T14:52:05,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768785594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768785596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768785595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742025_1201 (size=48639) 2024-11-16T14:52:05,615 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/74fa9c5061494727853f1e2e19b5a2ed 2024-11-16T14:52:05,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/18bbed52517d49f4bccc91bcd163955a is 50, key is test_row_0/B:col10/1731768725554/Put/seqid=0 2024-11-16T14:52:05,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742026_1202 (size=12301) 2024-11-16T14:52:05,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/18bbed52517d49f4bccc91bcd163955a 2024-11-16T14:52:05,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/6a41310c7331470f9b533734cbc9b2e2 is 50, key is test_row_0/C:col10/1731768725554/Put/seqid=0 2024-11-16T14:52:05,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742027_1203 (size=12301) 2024-11-16T14:52:05,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768785710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768785710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768785710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768785920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768785920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768785920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/6a41310c7331470f9b533734cbc9b2e2 2024-11-16T14:52:06,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/74fa9c5061494727853f1e2e19b5a2ed as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/74fa9c5061494727853f1e2e19b5a2ed 2024-11-16T14:52:06,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/74fa9c5061494727853f1e2e19b5a2ed, entries=250, sequenceid=324, filesize=47.5 K 2024-11-16T14:52:06,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/18bbed52517d49f4bccc91bcd163955a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/18bbed52517d49f4bccc91bcd163955a 2024-11-16T14:52:06,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/18bbed52517d49f4bccc91bcd163955a, entries=150, sequenceid=324, filesize=12.0 K 2024-11-16T14:52:06,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/6a41310c7331470f9b533734cbc9b2e2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/6a41310c7331470f9b533734cbc9b2e2 2024-11-16T14:52:06,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/6a41310c7331470f9b533734cbc9b2e2, entries=150, sequenceid=324, filesize=12.0 K 2024-11-16T14:52:06,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 72d0f116905856f1457e3e7f2ce74a1a in 524ms, sequenceid=324, compaction requested=true 2024-11-16T14:52:06,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:06,080 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:06,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:06,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:06,080 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:06,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:06,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:06,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72d0f116905856f1457e3e7f2ce74a1a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:06,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:06,082 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:06,082 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/B is initiating minor compaction (all files) 2024-11-16T14:52:06,082 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:06,082 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/B in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:06,082 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/A is initiating minor compaction (all files) 2024-11-16T14:52:06,082 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/A in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:06,082 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/d1e60a907f6c409da2860f93f5f471b8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ef2ec79b5dde4ddcb6457881a4fe832c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ac6094a81bf347da8934fd263bd83e26, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/18bbed52517d49f4bccc91bcd163955a] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=48.6 K 2024-11-16T14:52:06,082 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/fa01bc469b5c4a068a3600d499a0594d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/7a7ee45d2e6149cf8a1f45ea54476ce2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/622e78c644e44c408de1139891b158ed, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/74fa9c5061494727853f1e2e19b5a2ed] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=148.1 K 2024-11-16T14:52:06,082 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:06,082 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/fa01bc469b5c4a068a3600d499a0594d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/7a7ee45d2e6149cf8a1f45ea54476ce2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/622e78c644e44c408de1139891b158ed, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/74fa9c5061494727853f1e2e19b5a2ed] 2024-11-16T14:52:06,082 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e60a907f6c409da2860f93f5f471b8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731768722328 2024-11-16T14:52:06,082 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa01bc469b5c4a068a3600d499a0594d, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731768722328 2024-11-16T14:52:06,083 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ef2ec79b5dde4ddcb6457881a4fe832c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731768722377 2024-11-16T14:52:06,083 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a7ee45d2e6149cf8a1f45ea54476ce2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731768722377 2024-11-16T14:52:06,083 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6094a81bf347da8934fd263bd83e26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731768722735 2024-11-16T14:52:06,083 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 622e78c644e44c408de1139891b158ed, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731768722735 2024-11-16T14:52:06,083 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 18bbed52517d49f4bccc91bcd163955a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731768724912 2024-11-16T14:52:06,084 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74fa9c5061494727853f1e2e19b5a2ed, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731768724912 2024-11-16T14:52:06,090 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:06,093 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#B#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:06,094 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/b582e9ea718f4074a6c5e3de460a7370 is 50, key is test_row_0/B:col10/1731768725554/Put/seqid=0 2024-11-16T14:52:06,094 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116b77e77b0fc5d4816bd5ba89d32d16658_72d0f116905856f1457e3e7f2ce74a1a store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:06,097 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116b77e77b0fc5d4816bd5ba89d32d16658_72d0f116905856f1457e3e7f2ce74a1a, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:06,097 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b77e77b0fc5d4816bd5ba89d32d16658_72d0f116905856f1457e3e7f2ce74a1a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:06,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742028_1204 (size=12983) 2024-11-16T14:52:06,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742029_1205 (size=4469) 2024-11-16T14:52:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:06,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T14:52:06,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:52:06,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:06,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:52:06,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:06,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:52:06,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:06,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167ed3ff278f634649a1b30ac2e5e9c388_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768725593/Put/seqid=0 2024-11-16T14:52:06,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742030_1206 (size=17534) 2024-11-16T14:52:06,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768786251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768786252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768786257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768786361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768786370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768786370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,501 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#A#compaction#168 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:06,501 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/4146ca0c5c714ad3ababcf886d298b16 is 175, key is test_row_0/A:col10/1731768725554/Put/seqid=0 2024-11-16T14:52:06,503 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/b582e9ea718f4074a6c5e3de460a7370 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/b582e9ea718f4074a6c5e3de460a7370 2024-11-16T14:52:06,508 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/B of 72d0f116905856f1457e3e7f2ce74a1a into b582e9ea718f4074a6c5e3de460a7370(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:06,508 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:06,508 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/B, priority=12, startTime=1731768726080; duration=0sec 2024-11-16T14:52:06,508 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:06,508 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:B 2024-11-16T14:52:06,508 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:06,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742031_1207 (size=31937) 2024-11-16T14:52:06,511 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:06,511 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 72d0f116905856f1457e3e7f2ce74a1a/C is initiating minor compaction (all files) 2024-11-16T14:52:06,511 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72d0f116905856f1457e3e7f2ce74a1a/C in TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:06,511 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a95f6d3142d4e1195e84f2e5e375d71, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/8d50b4b2e7724f3a807868d59cce8460, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/1c761bab9c34408e82e3371a8ffb2a49, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/6a41310c7331470f9b533734cbc9b2e2] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp, totalSize=48.6 K 2024-11-16T14:52:06,512 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a95f6d3142d4e1195e84f2e5e375d71, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731768722328 2024-11-16T14:52:06,512 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d50b4b2e7724f3a807868d59cce8460, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731768722377 2024-11-16T14:52:06,512 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c761bab9c34408e82e3371a8ffb2a49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731768722735 2024-11-16T14:52:06,513 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a41310c7331470f9b533734cbc9b2e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731768724912 2024-11-16T14:52:06,516 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/4146ca0c5c714ad3ababcf886d298b16 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/4146ca0c5c714ad3ababcf886d298b16 2024-11-16T14:52:06,522 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/A of 72d0f116905856f1457e3e7f2ce74a1a into 4146ca0c5c714ad3ababcf886d298b16(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:06,522 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:06,522 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/A, priority=12, startTime=1731768726079; duration=0sec 2024-11-16T14:52:06,523 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:06,523 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:A 2024-11-16T14:52:06,529 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72d0f116905856f1457e3e7f2ce74a1a#C#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:06,530 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/2c5fa9e0214a4bb49015403367a679e3 is 50, key is test_row_0/C:col10/1731768725554/Put/seqid=0 2024-11-16T14:52:06,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742032_1208 (size=12983) 2024-11-16T14:52:06,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768786571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768786581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768786581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,646 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:06,650 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167ed3ff278f634649a1b30ac2e5e9c388_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167ed3ff278f634649a1b30ac2e5e9c388_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:06,651 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/8de8965a6a8b481286b4caab88eccfa1, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:06,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/8de8965a6a8b481286b4caab88eccfa1 is 175, key is test_row_0/A:col10/1731768725593/Put/seqid=0 2024-11-16T14:52:06,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742033_1209 (size=48639) 2024-11-16T14:52:06,655 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/8de8965a6a8b481286b4caab88eccfa1 2024-11-16T14:52:06,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/758ec82222b74f6fba27c5a1acdf1782 is 50, key is test_row_0/B:col10/1731768725593/Put/seqid=0 2024-11-16T14:52:06,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742034_1210 (size=12301) 2024-11-16T14:52:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T14:52:06,882 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:06,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:06,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-16T14:52:06,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768786881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,884 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:06,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:52:06,885 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:06,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768786891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768786891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:06,912 INFO [Thread-632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-632. 2024-11-16T14:52:06,912 DEBUG [Thread-632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:06,912 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:06,912 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:06,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:06,914 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-16T14:52:06,914 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:06,914 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:06,914 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:06,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:06,917 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-16T14:52:06,917 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:06,917 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:06,918 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:06,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:06,928 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-16T14:52:06,928 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:06,928 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:06,928 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:06,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:06,945 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/2c5fa9e0214a4bb49015403367a679e3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/2c5fa9e0214a4bb49015403367a679e3 2024-11-16T14:52:06,950 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72d0f116905856f1457e3e7f2ce74a1a/C of 72d0f116905856f1457e3e7f2ce74a1a into 2c5fa9e0214a4bb49015403367a679e3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:06,950 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:06,950 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., storeName=72d0f116905856f1457e3e7f2ce74a1a/C, priority=12, startTime=1731768726080; duration=0sec 2024-11-16T14:52:06,950 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:06,950 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72d0f116905856f1457e3e7f2ce74a1a:C 2024-11-16T14:52:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:52:07,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:52:07,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:07,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:07,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:07,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/758ec82222b74f6fba27c5a1acdf1782 2024-11-16T14:52:07,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/da6e7f8b73164013b70185cb8593d047 is 50, key is test_row_0/C:col10/1731768725593/Put/seqid=0 2024-11-16T14:52:07,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742035_1211 (size=12301) 2024-11-16T14:52:07,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:52:07,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:07,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:07,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:07,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:52:07,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:52:07,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:07,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:07,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:07,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53928 deadline: 1731768787391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:07,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53904 deadline: 1731768787401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:07,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53920 deadline: 1731768787410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:07,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:07,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:07,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:07,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/da6e7f8b73164013b70185cb8593d047 2024-11-16T14:52:07,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/8de8965a6a8b481286b4caab88eccfa1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8de8965a6a8b481286b4caab88eccfa1 2024-11-16T14:52:07,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8de8965a6a8b481286b4caab88eccfa1, entries=250, sequenceid=342, filesize=47.5 K 2024-11-16T14:52:07,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/758ec82222b74f6fba27c5a1acdf1782 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/758ec82222b74f6fba27c5a1acdf1782 2024-11-16T14:52:07,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/758ec82222b74f6fba27c5a1acdf1782, entries=150, sequenceid=342, filesize=12.0 K 2024-11-16T14:52:07,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/da6e7f8b73164013b70185cb8593d047 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/da6e7f8b73164013b70185cb8593d047 2024-11-16T14:52:07,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:52:07,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:07,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:07,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:07,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:07,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/da6e7f8b73164013b70185cb8593d047, entries=150, sequenceid=342, filesize=12.0 K 2024-11-16T14:52:07,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 72d0f116905856f1457e3e7f2ce74a1a in 1268ms, sequenceid=342, compaction requested=false 2024-11-16T14:52:07,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:52:07,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T14:52:07,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:07,651 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:52:07,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:52:07,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:07,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:52:07,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:07,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:52:07,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116762a165a6809461c9fefe43f1b53ff07_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768726256/Put/seqid=0 2024-11-16T14:52:07,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742036_1212 (size=12454) 2024-11-16T14:52:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:52:08,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:08,068 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116762a165a6809461c9fefe43f1b53ff07_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116762a165a6809461c9fefe43f1b53ff07_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:08,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/01337ac5ee8d411c866ac4e1d23f039c, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:08,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/01337ac5ee8d411c866ac4e1d23f039c is 175, key is test_row_0/A:col10/1731768726256/Put/seqid=0 2024-11-16T14:52:08,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742037_1213 (size=31255) 2024-11-16T14:52:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:08,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. as already flushing 2024-11-16T14:52:08,402 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-16T14:52:08,402 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:08,402 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:08,402 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:08,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:08,412 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-11-16T14:52:08,412 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:08,412 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:08,412 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:08,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:08,422 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-16T14:52:08,422 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:08,422 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:08,422 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:08,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:08,477 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/01337ac5ee8d411c866ac4e1d23f039c 2024-11-16T14:52:08,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/9f83d91bfb2148d8a1bea96811be4a50 is 50, key is test_row_0/B:col10/1731768726256/Put/seqid=0 2024-11-16T14:52:08,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742038_1214 (size=12301) 2024-11-16T14:52:08,890 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/9f83d91bfb2148d8a1bea96811be4a50 2024-11-16T14:52:08,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/e9a445b5420d4593b6a179d3af72207f is 50, key is test_row_0/C:col10/1731768726256/Put/seqid=0 2024-11-16T14:52:08,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742039_1215 (size=12301) 2024-11-16T14:52:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:52:09,307 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/e9a445b5420d4593b6a179d3af72207f 2024-11-16T14:52:09,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/01337ac5ee8d411c866ac4e1d23f039c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/01337ac5ee8d411c866ac4e1d23f039c 2024-11-16T14:52:09,317 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/01337ac5ee8d411c866ac4e1d23f039c, entries=150, sequenceid=359, filesize=30.5 K 2024-11-16T14:52:09,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/9f83d91bfb2148d8a1bea96811be4a50 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/9f83d91bfb2148d8a1bea96811be4a50 2024-11-16T14:52:09,323 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/9f83d91bfb2148d8a1bea96811be4a50, entries=150, sequenceid=359, filesize=12.0 K 2024-11-16T14:52:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/e9a445b5420d4593b6a179d3af72207f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/e9a445b5420d4593b6a179d3af72207f 2024-11-16T14:52:09,329 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/e9a445b5420d4593b6a179d3af72207f, entries=150, sequenceid=359, filesize=12.0 K 2024-11-16T14:52:09,330 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for 72d0f116905856f1457e3e7f2ce74a1a in 1679ms, sequenceid=359, compaction requested=true 2024-11-16T14:52:09,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 72d0f116905856f1457e3e7f2ce74a1a: 2024-11-16T14:52:09,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:09,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-16T14:52:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-16T14:52:09,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-16T14:52:09,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4470 sec 2024-11-16T14:52:09,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.4520 sec 2024-11-16T14:52:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T14:52:11,051 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:11,710 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:52:15,113 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-11-16T14:52:15,113 INFO [Thread-621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-621. 2024-11-16T14:52:15,113 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:15,113 DEBUG [Thread-621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:15,113 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:15,113 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:15,113 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:15,113 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:15,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:15,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2991 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2777 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1163 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3489 rows 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1155 2024-11-16T14:52:15,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3465 rows 2024-11-16T14:52:15,114 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:52:15,114 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:52:15,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:15,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:15,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:15,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:52:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:52:15,121 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768735120"}]},"ts":"1731768735120"} 2024-11-16T14:52:15,123 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:52:15,123 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:52:15,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:52:15,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, UNASSIGN}] 2024-11-16T14:52:15,128 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, UNASSIGN 2024-11-16T14:52:15,129 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=72d0f116905856f1457e3e7f2ce74a1a, regionState=CLOSING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:15,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, UNASSIGN because future has completed 2024-11-16T14:52:15,131 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:52:15,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:52:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:52:15,284 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:15,284 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:52:15,284 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 72d0f116905856f1457e3e7f2ce74a1a, disabling compactions & flushes 2024-11-16T14:52:15,284 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:15,284 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:15,285 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. after waiting 0 ms 2024-11-16T14:52:15,285 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:15,285 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 72d0f116905856f1457e3e7f2ce74a1a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T14:52:15,285 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=A 2024-11-16T14:52:15,286 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:15,286 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=B 2024-11-16T14:52:15,286 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:15,286 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72d0f116905856f1457e3e7f2ce74a1a, store=C 2024-11-16T14:52:15,286 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:15,297 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167c574ce5f4cf4927af2a89d1f6af1c70_72d0f116905856f1457e3e7f2ce74a1a is 50, key is test_row_0/A:col10/1731768728400/Put/seqid=0 2024-11-16T14:52:15,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742040_1216 (size=12454) 2024-11-16T14:52:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:52:15,702 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:15,709 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167c574ce5f4cf4927af2a89d1f6af1c70_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167c574ce5f4cf4927af2a89d1f6af1c70_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:15,710 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/da5a56a9dd384953a1db8c1f3a4154bb, store: [table=TestAcidGuarantees family=A region=72d0f116905856f1457e3e7f2ce74a1a] 2024-11-16T14:52:15,711 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/da5a56a9dd384953a1db8c1f3a4154bb is 175, key is test_row_0/A:col10/1731768728400/Put/seqid=0 2024-11-16T14:52:15,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742041_1217 (size=31255) 2024-11-16T14:52:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:52:16,117 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/da5a56a9dd384953a1db8c1f3a4154bb 2024-11-16T14:52:16,130 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/3f891f384a7d4a96bed70b1ce07157ec is 50, key is test_row_0/B:col10/1731768728400/Put/seqid=0 2024-11-16T14:52:16,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742042_1218 (size=12301) 2024-11-16T14:52:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:52:16,536 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/3f891f384a7d4a96bed70b1ce07157ec 2024-11-16T14:52:16,552 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/df2904d038394438868a78bd170841ab is 50, key is test_row_0/C:col10/1731768728400/Put/seqid=0 2024-11-16T14:52:16,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742043_1219 (size=12301) 2024-11-16T14:52:16,956 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/df2904d038394438868a78bd170841ab 2024-11-16T14:52:16,961 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/A/da5a56a9dd384953a1db8c1f3a4154bb as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/da5a56a9dd384953a1db8c1f3a4154bb 2024-11-16T14:52:16,965 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/da5a56a9dd384953a1db8c1f3a4154bb, entries=150, sequenceid=367, filesize=30.5 K 2024-11-16T14:52:16,966 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/B/3f891f384a7d4a96bed70b1ce07157ec as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3f891f384a7d4a96bed70b1ce07157ec 2024-11-16T14:52:16,971 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3f891f384a7d4a96bed70b1ce07157ec, entries=150, sequenceid=367, filesize=12.0 K 2024-11-16T14:52:16,972 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/.tmp/C/df2904d038394438868a78bd170841ab as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/df2904d038394438868a78bd170841ab 2024-11-16T14:52:16,976 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/df2904d038394438868a78bd170841ab, entries=150, sequenceid=367, filesize=12.0 K 2024-11-16T14:52:16,977 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 72d0f116905856f1457e3e7f2ce74a1a in 1692ms, sequenceid=367, compaction requested=true 2024-11-16T14:52:16,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a53d9442d0a14285aa4395b7480d660e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/9d2184aad52d416782824a0b3ed0093a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/84fc0deb4db344318ccc15f498f0e54a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/f16a875f0c88466691f65093c07da06a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/16fcaf71259445c68610fb6ae114cbb3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/3aa14e376b184b61a5f35adf6843033b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/1056d70e7aea4b93ac081d0572bcce8e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a9a8968a3f6c4c89a0f6d11054e9db93, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/4393393e686242c794dd579be0bf20b5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8d9e8ea064574ab199ab32f41801031e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/719834477e0246edbcb6487f65bf0be4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/09433ba2bbbe4ef58a88996b5c39db1e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/71d137da0b8a4f47aad76955eec4df02, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/c2a53b624bd54e51ab85a4246bc2467e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/0779be35e26046cc83f1259af4f121e2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/fa01bc469b5c4a068a3600d499a0594d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/d349fca05d864da9a46328d25dc4462b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/7a7ee45d2e6149cf8a1f45ea54476ce2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/622e78c644e44c408de1139891b158ed, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/74fa9c5061494727853f1e2e19b5a2ed] to archive 2024-11-16T14:52:16,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:52:16,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a53d9442d0a14285aa4395b7480d660e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a53d9442d0a14285aa4395b7480d660e 2024-11-16T14:52:16,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/9d2184aad52d416782824a0b3ed0093a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/9d2184aad52d416782824a0b3ed0093a 2024-11-16T14:52:16,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/84fc0deb4db344318ccc15f498f0e54a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/84fc0deb4db344318ccc15f498f0e54a 2024-11-16T14:52:16,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/f16a875f0c88466691f65093c07da06a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/f16a875f0c88466691f65093c07da06a 2024-11-16T14:52:16,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/16fcaf71259445c68610fb6ae114cbb3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/16fcaf71259445c68610fb6ae114cbb3 2024-11-16T14:52:16,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/3aa14e376b184b61a5f35adf6843033b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/3aa14e376b184b61a5f35adf6843033b 2024-11-16T14:52:16,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/1056d70e7aea4b93ac081d0572bcce8e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/1056d70e7aea4b93ac081d0572bcce8e 2024-11-16T14:52:16,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a9a8968a3f6c4c89a0f6d11054e9db93 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/a9a8968a3f6c4c89a0f6d11054e9db93 2024-11-16T14:52:16,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/4393393e686242c794dd579be0bf20b5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/4393393e686242c794dd579be0bf20b5 2024-11-16T14:52:16,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8d9e8ea064574ab199ab32f41801031e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8d9e8ea064574ab199ab32f41801031e 2024-11-16T14:52:16,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/719834477e0246edbcb6487f65bf0be4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/719834477e0246edbcb6487f65bf0be4 2024-11-16T14:52:16,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/09433ba2bbbe4ef58a88996b5c39db1e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/09433ba2bbbe4ef58a88996b5c39db1e 2024-11-16T14:52:16,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/71d137da0b8a4f47aad76955eec4df02 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/71d137da0b8a4f47aad76955eec4df02 2024-11-16T14:52:16,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/c2a53b624bd54e51ab85a4246bc2467e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/c2a53b624bd54e51ab85a4246bc2467e 2024-11-16T14:52:16,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/0779be35e26046cc83f1259af4f121e2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/0779be35e26046cc83f1259af4f121e2 2024-11-16T14:52:16,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/fa01bc469b5c4a068a3600d499a0594d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/fa01bc469b5c4a068a3600d499a0594d 2024-11-16T14:52:16,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/d349fca05d864da9a46328d25dc4462b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/d349fca05d864da9a46328d25dc4462b 2024-11-16T14:52:16,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/7a7ee45d2e6149cf8a1f45ea54476ce2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/7a7ee45d2e6149cf8a1f45ea54476ce2 2024-11-16T14:52:16,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/622e78c644e44c408de1139891b158ed to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/622e78c644e44c408de1139891b158ed 2024-11-16T14:52:17,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/74fa9c5061494727853f1e2e19b5a2ed to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/74fa9c5061494727853f1e2e19b5a2ed 2024-11-16T14:52:17,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/41dbce106cbd46508273c1c5595a18a3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f1612f41603e44bfaa8e2122f7871489, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ce2ff3d9d6d5492782d18cfc85496afb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/22972ce4669645f2ac013f2a781f729c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/75a55d77da2d467cb106da49dabf0c13, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/baa685051b7f43618f480d801b6e82c3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/23e3d75969314f378307f59f8fb9e034, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3d406d417396483db0a887fb58538850, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/09b1587a5907423380632e3e2e402237, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f2579ce41f5b49bb83f25942e4d1db69, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/261174f9a23a413ab3ea8c58d1de36fc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3a132061958e42acb1054a6701c542da, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/0498f020c7924a0b9a1f710e2c4a2541, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f858442f4aa14f918aff61101f4f9abb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/b90f3988b3e946a890df09c66e786c2a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/d1e60a907f6c409da2860f93f5f471b8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/627f16852acb48d3bd752db5ee181758, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ef2ec79b5dde4ddcb6457881a4fe832c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ac6094a81bf347da8934fd263bd83e26, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/18bbed52517d49f4bccc91bcd163955a] to archive 2024-11-16T14:52:17,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:52:17,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/41dbce106cbd46508273c1c5595a18a3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/41dbce106cbd46508273c1c5595a18a3 2024-11-16T14:52:17,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f1612f41603e44bfaa8e2122f7871489 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f1612f41603e44bfaa8e2122f7871489 2024-11-16T14:52:17,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ce2ff3d9d6d5492782d18cfc85496afb to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ce2ff3d9d6d5492782d18cfc85496afb 2024-11-16T14:52:17,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/22972ce4669645f2ac013f2a781f729c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/22972ce4669645f2ac013f2a781f729c 2024-11-16T14:52:17,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/75a55d77da2d467cb106da49dabf0c13 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/75a55d77da2d467cb106da49dabf0c13 2024-11-16T14:52:17,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/baa685051b7f43618f480d801b6e82c3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/baa685051b7f43618f480d801b6e82c3 2024-11-16T14:52:17,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/23e3d75969314f378307f59f8fb9e034 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/23e3d75969314f378307f59f8fb9e034 2024-11-16T14:52:17,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3d406d417396483db0a887fb58538850 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3d406d417396483db0a887fb58538850 2024-11-16T14:52:17,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/09b1587a5907423380632e3e2e402237 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/09b1587a5907423380632e3e2e402237 2024-11-16T14:52:17,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f2579ce41f5b49bb83f25942e4d1db69 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f2579ce41f5b49bb83f25942e4d1db69 2024-11-16T14:52:17,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/261174f9a23a413ab3ea8c58d1de36fc to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/261174f9a23a413ab3ea8c58d1de36fc 2024-11-16T14:52:17,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3a132061958e42acb1054a6701c542da to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3a132061958e42acb1054a6701c542da 2024-11-16T14:52:17,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/0498f020c7924a0b9a1f710e2c4a2541 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/0498f020c7924a0b9a1f710e2c4a2541 2024-11-16T14:52:17,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f858442f4aa14f918aff61101f4f9abb to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/f858442f4aa14f918aff61101f4f9abb 2024-11-16T14:52:17,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/b90f3988b3e946a890df09c66e786c2a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/b90f3988b3e946a890df09c66e786c2a 2024-11-16T14:52:17,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/d1e60a907f6c409da2860f93f5f471b8 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/d1e60a907f6c409da2860f93f5f471b8 2024-11-16T14:52:17,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/627f16852acb48d3bd752db5ee181758 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/627f16852acb48d3bd752db5ee181758 2024-11-16T14:52:17,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ef2ec79b5dde4ddcb6457881a4fe832c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ef2ec79b5dde4ddcb6457881a4fe832c 2024-11-16T14:52:17,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ac6094a81bf347da8934fd263bd83e26 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/ac6094a81bf347da8934fd263bd83e26 2024-11-16T14:52:17,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/18bbed52517d49f4bccc91bcd163955a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/18bbed52517d49f4bccc91bcd163955a 2024-11-16T14:52:17,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/c0ef15ec75be44e0b84bc685b8e79b79, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a38dd4a291742e5b017a88c70d4e236, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b6e5b50c7eb6485981cff3a98f0fc70b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/fcd547d414cd4edfa2bdf8ebf1f9432f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e20658ed6a54e4396c6c92adda22b4e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d87c505e2d5f4255997981cb3174c352, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/7e39b85cb342451bb5434917a5b48ed7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a5e0670e16f145d68447c78bcacd73c6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d1f23ad359ce4b26a5f71e7d4ca2c453, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/7d2bd9f951a4457fa6664fc5257f8ea6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b8d76b9240184dcf90d88a8101103969, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/cc916b303241405ebdd2a33b7a85472e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a0ccd12cd34d485b8a862d4ba24ebe30, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d6658c9d250440ef886f846e9b493744, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e8955395bee47bda56522151531f07c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a95f6d3142d4e1195e84f2e5e375d71, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/0bdcbdbee9c84fa4a9fe4bd7d5d0cb1e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/8d50b4b2e7724f3a807868d59cce8460, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/1c761bab9c34408e82e3371a8ffb2a49, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/6a41310c7331470f9b533734cbc9b2e2] to archive 2024-11-16T14:52:17,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:52:17,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/c0ef15ec75be44e0b84bc685b8e79b79 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/c0ef15ec75be44e0b84bc685b8e79b79 2024-11-16T14:52:17,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a38dd4a291742e5b017a88c70d4e236 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a38dd4a291742e5b017a88c70d4e236 2024-11-16T14:52:17,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b6e5b50c7eb6485981cff3a98f0fc70b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b6e5b50c7eb6485981cff3a98f0fc70b 2024-11-16T14:52:17,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/fcd547d414cd4edfa2bdf8ebf1f9432f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/fcd547d414cd4edfa2bdf8ebf1f9432f 2024-11-16T14:52:17,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e20658ed6a54e4396c6c92adda22b4e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e20658ed6a54e4396c6c92adda22b4e 2024-11-16T14:52:17,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d87c505e2d5f4255997981cb3174c352 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d87c505e2d5f4255997981cb3174c352 2024-11-16T14:52:17,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/7e39b85cb342451bb5434917a5b48ed7 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/7e39b85cb342451bb5434917a5b48ed7 2024-11-16T14:52:17,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a5e0670e16f145d68447c78bcacd73c6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a5e0670e16f145d68447c78bcacd73c6 2024-11-16T14:52:17,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d1f23ad359ce4b26a5f71e7d4ca2c453 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d1f23ad359ce4b26a5f71e7d4ca2c453 2024-11-16T14:52:17,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/7d2bd9f951a4457fa6664fc5257f8ea6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/7d2bd9f951a4457fa6664fc5257f8ea6 2024-11-16T14:52:17,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b8d76b9240184dcf90d88a8101103969 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/b8d76b9240184dcf90d88a8101103969 2024-11-16T14:52:17,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/cc916b303241405ebdd2a33b7a85472e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/cc916b303241405ebdd2a33b7a85472e 2024-11-16T14:52:17,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a0ccd12cd34d485b8a862d4ba24ebe30 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/a0ccd12cd34d485b8a862d4ba24ebe30 2024-11-16T14:52:17,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d6658c9d250440ef886f846e9b493744 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/d6658c9d250440ef886f846e9b493744 2024-11-16T14:52:17,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e8955395bee47bda56522151531f07c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/3e8955395bee47bda56522151531f07c 2024-11-16T14:52:17,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a95f6d3142d4e1195e84f2e5e375d71 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/4a95f6d3142d4e1195e84f2e5e375d71 2024-11-16T14:52:17,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/0bdcbdbee9c84fa4a9fe4bd7d5d0cb1e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/0bdcbdbee9c84fa4a9fe4bd7d5d0cb1e 2024-11-16T14:52:17,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/8d50b4b2e7724f3a807868d59cce8460 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/8d50b4b2e7724f3a807868d59cce8460 2024-11-16T14:52:17,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/1c761bab9c34408e82e3371a8ffb2a49 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/1c761bab9c34408e82e3371a8ffb2a49 2024-11-16T14:52:17,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/6a41310c7331470f9b533734cbc9b2e2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/6a41310c7331470f9b533734cbc9b2e2 2024-11-16T14:52:17,050 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/recovered.edits/370.seqid, newMaxSeqId=370, maxSeqId=4 2024-11-16T14:52:17,050 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. 2024-11-16T14:52:17,050 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 72d0f116905856f1457e3e7f2ce74a1a: Waiting for close lock at 1731768735284Running coprocessor pre-close hooks at 1731768735284Disabling compacts and flushes for region at 1731768735284Disabling writes for close at 1731768735285 (+1 ms)Obtaining lock to block concurrent updates at 1731768735285Preparing flush snapshotting stores in 72d0f116905856f1457e3e7f2ce74a1a at 1731768735285Finished memstore snapshotting TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731768735286 (+1 ms)Flushing stores of TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a. at 1731768735287 (+1 ms)Flushing 72d0f116905856f1457e3e7f2ce74a1a/A: creating writer at 1731768735288 (+1 ms)Flushing mob file 72d0f116905856f1457e3e7f2ce74a1a/A: appending metadata at 1731768735297 (+9 ms)Flushing mob file 72d0f116905856f1457e3e7f2ce74a1a/A: closing flushed file at 1731768735297Flushing 72d0f116905856f1457e3e7f2ce74a1a/A: appending metadata at 1731768735710 (+413 ms)Flushing 72d0f116905856f1457e3e7f2ce74a1a/A: closing flushed file at 1731768735710Flushing 72d0f116905856f1457e3e7f2ce74a1a/B: creating writer at 1731768736126 (+416 ms)Flushing 72d0f116905856f1457e3e7f2ce74a1a/B: appending metadata at 1731768736130 (+4 ms)Flushing 72d0f116905856f1457e3e7f2ce74a1a/B: closing flushed file at 1731768736130Flushing 72d0f116905856f1457e3e7f2ce74a1a/C: creating writer at 1731768736548 (+418 ms)Flushing 72d0f116905856f1457e3e7f2ce74a1a/C: appending metadata at 1731768736551 (+3 ms)Flushing 72d0f116905856f1457e3e7f2ce74a1a/C: closing flushed file at 1731768736551Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a3d14f9: reopening flushed file at 1731768736960 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78a02893: reopening flushed file at 1731768736965 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54dc8b80: reopening flushed file at 1731768736971 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 72d0f116905856f1457e3e7f2ce74a1a in 1692ms, sequenceid=367, compaction requested=true at 1731768736977 (+6 ms)Writing region close event to WAL at 1731768737047 (+70 ms)Running coprocessor post-close hooks at 1731768737050 (+3 ms)Closed at 1731768737050 2024-11-16T14:52:17,052 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,053 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=72d0f116905856f1457e3e7f2ce74a1a, regionState=CLOSED 2024-11-16T14:52:17,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:52:17,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-16T14:52:17,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 72d0f116905856f1457e3e7f2ce74a1a, server=fc3fda29f660,43955,1731768673386 in 1.9240 sec 2024-11-16T14:52:17,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-16T14:52:17,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72d0f116905856f1457e3e7f2ce74a1a, UNASSIGN in 1.9290 sec 2024-11-16T14:52:17,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-16T14:52:17,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9330 sec 2024-11-16T14:52:17,060 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768737060"}]},"ts":"1731768737060"} 2024-11-16T14:52:17,061 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:52:17,061 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:52:17,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9450 sec 2024-11-16T14:52:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T14:52:17,282 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:17,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:52:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:17,287 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T14:52:17,289 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:17,292 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,295 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/recovered.edits] 2024-11-16T14:52:17,298 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/01337ac5ee8d411c866ac4e1d23f039c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/01337ac5ee8d411c866ac4e1d23f039c 2024-11-16T14:52:17,299 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/4146ca0c5c714ad3ababcf886d298b16 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/4146ca0c5c714ad3ababcf886d298b16 2024-11-16T14:52:17,300 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8de8965a6a8b481286b4caab88eccfa1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/8de8965a6a8b481286b4caab88eccfa1 2024-11-16T14:52:17,301 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/da5a56a9dd384953a1db8c1f3a4154bb to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/A/da5a56a9dd384953a1db8c1f3a4154bb 2024-11-16T14:52:17,304 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3f891f384a7d4a96bed70b1ce07157ec to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/3f891f384a7d4a96bed70b1ce07157ec 2024-11-16T14:52:17,305 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/758ec82222b74f6fba27c5a1acdf1782 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/758ec82222b74f6fba27c5a1acdf1782 2024-11-16T14:52:17,306 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/9f83d91bfb2148d8a1bea96811be4a50 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/9f83d91bfb2148d8a1bea96811be4a50 2024-11-16T14:52:17,306 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/b582e9ea718f4074a6c5e3de460a7370 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/B/b582e9ea718f4074a6c5e3de460a7370 2024-11-16T14:52:17,308 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/2c5fa9e0214a4bb49015403367a679e3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/2c5fa9e0214a4bb49015403367a679e3 2024-11-16T14:52:17,309 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/da6e7f8b73164013b70185cb8593d047 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/da6e7f8b73164013b70185cb8593d047 2024-11-16T14:52:17,310 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/df2904d038394438868a78bd170841ab to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/df2904d038394438868a78bd170841ab 2024-11-16T14:52:17,311 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/e9a445b5420d4593b6a179d3af72207f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/C/e9a445b5420d4593b6a179d3af72207f 2024-11-16T14:52:17,314 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/recovered.edits/370.seqid to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a/recovered.edits/370.seqid 2024-11-16T14:52:17,314 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,314 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:52:17,314 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:52:17,315 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T14:52:17,318 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111602ed727afc75408687099881868ff0bc_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111602ed727afc75408687099881868ff0bc_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,319 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160ae78916654c43bd9f8b770c65272269_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160ae78916654c43bd9f8b770c65272269_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,320 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161cedf014358f47fa967b614e5d32b5f7_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161cedf014358f47fa967b614e5d32b5f7_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,321 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163b08533a7250413d85533b419640657d_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163b08533a7250413d85533b419640657d_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,322 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163efb1a8645a04005ba24829ce51a460d_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163efb1a8645a04005ba24829ce51a460d_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,323 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165ec9078948ba4dbcaa65eb2346dd8c8d_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165ec9078948ba4dbcaa65eb2346dd8c8d_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,324 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116686242da85d842e5a924eb8f7fb33693_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116686242da85d842e5a924eb8f7fb33693_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,325 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116762a165a6809461c9fefe43f1b53ff07_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116762a165a6809461c9fefe43f1b53ff07_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,325 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167c574ce5f4cf4927af2a89d1f6af1c70_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167c574ce5f4cf4927af2a89d1f6af1c70_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,326 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167ed3ff278f634649a1b30ac2e5e9c388_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167ed3ff278f634649a1b30ac2e5e9c388_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,327 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116869508d7792d4c7eacfdcdd98d25372c_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116869508d7792d4c7eacfdcdd98d25372c_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,328 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111697d3ae0eef02400984a7e9a3586cdeb6_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111697d3ae0eef02400984a7e9a3586cdeb6_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,329 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116989256963560421e9d6aed552d1de5ce_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116989256963560421e9d6aed552d1de5ce_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,329 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169fbf79f3d325415a944229cdaf0162fb_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169fbf79f3d325415a944229cdaf0162fb_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,330 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b58e9d5822db44e19a2babe411c4b9b3_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b58e9d5822db44e19a2babe411c4b9b3_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,331 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c1febb47ce1741698969931f0304d7c6_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c1febb47ce1741698969931f0304d7c6_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,332 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ea9c6803b7ef468fba399d69dca7459a_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ea9c6803b7ef468fba399d69dca7459a_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,333 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f75d84d1a8514a54bb9867871a4e4024_72d0f116905856f1457e3e7f2ce74a1a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f75d84d1a8514a54bb9867871a4e4024_72d0f116905856f1457e3e7f2ce74a1a 2024-11-16T14:52:17,333 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:52:17,335 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:17,337 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:52:17,339 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:52:17,340 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:17,340 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:52:17,340 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731768737340"}]},"ts":"9223372036854775807"} 2024-11-16T14:52:17,342 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:52:17,342 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 72d0f116905856f1457e3e7f2ce74a1a, NAME => 'TestAcidGuarantees,,1731768703706.72d0f116905856f1457e3e7f2ce74a1a.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:52:17,342 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:52:17,343 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731768737342"}]},"ts":"9223372036854775807"} 2024-11-16T14:52:17,345 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:52:17,345 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:17,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-16T14:52:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T14:52:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:52:17,401 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:17,409 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 249), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=373 (was 320) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5062 (was 4971) - AvailableMemoryMB LEAK? - 2024-11-16T14:52:17,417 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=373, ProcessCount=11, AvailableMemoryMB=5062 2024-11-16T14:52:17,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:52:17,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:52:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:17,420 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:52:17,421 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:17,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-16T14:52:17,421 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:52:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-16T14:52:17,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742044_1220 (size=1061) 2024-11-16T14:52:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-16T14:52:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-16T14:52:17,833 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d 2024-11-16T14:52:17,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742045_1221 (size=53) 2024-11-16T14:52:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-16T14:52:18,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:52:18,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0aa24bef827bbc1122cec8569b1d4491, disabling compactions & flushes 2024-11-16T14:52:18,244 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:18,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:18,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. after waiting 0 ms 2024-11-16T14:52:18,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:18,244 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:18,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0aa24bef827bbc1122cec8569b1d4491: Waiting for close lock at 1731768738244Disabling compacts and flushes for region at 1731768738244Disabling writes for close at 1731768738244Writing region close event to WAL at 1731768738244Closed at 1731768738244 2024-11-16T14:52:18,245 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:52:18,246 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731768738245"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731768738245"}]},"ts":"1731768738245"} 2024-11-16T14:52:18,247 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:52:18,248 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:52:18,248 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768738248"}]},"ts":"1731768738248"} 2024-11-16T14:52:18,249 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:52:18,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0aa24bef827bbc1122cec8569b1d4491, ASSIGN}] 2024-11-16T14:52:18,251 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0aa24bef827bbc1122cec8569b1d4491, ASSIGN 2024-11-16T14:52:18,252 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0aa24bef827bbc1122cec8569b1d4491, ASSIGN; state=OFFLINE, location=fc3fda29f660,43955,1731768673386; forceNewPlan=false, retain=false 2024-11-16T14:52:18,402 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=0aa24bef827bbc1122cec8569b1d4491, regionState=OPENING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:18,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0aa24bef827bbc1122cec8569b1d4491, ASSIGN because future has completed 2024-11-16T14:52:18,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:52:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-16T14:52:18,600 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:18,600 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:52:18,601 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,601 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:52:18,601 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,601 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,603 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,605 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:52:18,606 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0aa24bef827bbc1122cec8569b1d4491 columnFamilyName A 2024-11-16T14:52:18,606 DEBUG [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:18,607 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] regionserver.HStore(327): Store=0aa24bef827bbc1122cec8569b1d4491/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:52:18,607 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,609 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:52:18,609 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0aa24bef827bbc1122cec8569b1d4491 columnFamilyName B 2024-11-16T14:52:18,609 DEBUG [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:18,610 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] regionserver.HStore(327): Store=0aa24bef827bbc1122cec8569b1d4491/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:52:18,610 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,612 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:52:18,612 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0aa24bef827bbc1122cec8569b1d4491 columnFamilyName C 2024-11-16T14:52:18,612 DEBUG [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:18,612 INFO [StoreOpener-0aa24bef827bbc1122cec8569b1d4491-1 {}] regionserver.HStore(327): Store=0aa24bef827bbc1122cec8569b1d4491/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:52:18,613 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:18,613 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,613 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,614 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,615 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,615 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,615 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:52:18,617 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,619 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:52:18,619 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 0aa24bef827bbc1122cec8569b1d4491; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69451059, jitterRate=0.03490142524242401}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:52:18,619 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:18,620 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 0aa24bef827bbc1122cec8569b1d4491: Running coprocessor pre-open hook at 1731768738601Writing region info on filesystem at 1731768738601Initializing all the Stores at 1731768738603 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768738603Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768738603Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768738603Cleaning up temporary data from old regions at 1731768738615 (+12 ms)Running coprocessor post-open hooks at 1731768738619 (+4 ms)Region opened successfully at 1731768738620 (+1 ms) 2024-11-16T14:52:18,621 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., pid=61, masterSystemTime=1731768738596 2024-11-16T14:52:18,623 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:18,623 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:18,623 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=0aa24bef827bbc1122cec8569b1d4491, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:18,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:52:18,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-16T14:52:18,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 in 219 msec 2024-11-16T14:52:18,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-16T14:52:18,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0aa24bef827bbc1122cec8569b1d4491, ASSIGN in 377 msec 2024-11-16T14:52:18,629 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:52:18,629 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768738629"}]},"ts":"1731768738629"} 2024-11-16T14:52:18,630 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:52:18,631 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:52:18,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2120 sec 2024-11-16T14:52:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-16T14:52:19,572 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:19,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bdc5a1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,577 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7b588d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,579 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc56c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,580 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,582 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:52:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fc3fda29f660,35823,1731768672605 2024-11-16T14:52:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@695dbfab 2024-11-16T14:52:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:52:19,584 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:52:19,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20832dd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,586 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7a3ce7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,587 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad9e6ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb66027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,589 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fe91b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,591 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22dd2a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b4a085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,593 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db2b8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,594 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e94da6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f8f105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,596 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1e4181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,597 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc7c7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aaa706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,599 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f50f32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,600 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e39179e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417b74fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,602 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792d36d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,603 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@413f4e54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0a17bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,605 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66ad0753, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,606 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbfee70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb595da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,607 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,608 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d69a3b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,609 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce3d992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@262694dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,612 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad607ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,613 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1735c82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df95348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:19,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:19,614 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585857b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:19,616 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@321dec87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:19,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:19,617 DEBUG [Thread-1049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,617 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,617 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,617 DEBUG [Thread-1051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,618 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,618 DEBUG [Thread-1047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,618 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-16T14:52:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,619 DEBUG [Thread-1045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,620 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,620 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,621 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:19,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:52:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:52:19,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:52:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:52:19,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T14:52:19,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:52:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:52:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:52:19,628 DEBUG [Thread-1062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:52:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:52:19,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:19,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:52:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:19,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:52:19,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:19,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:19,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:19,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:19,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:19,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:19,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/34b47461cbd64364a095c46a9bf28aaf is 50, key is test_row_0/A:col10/1731768739632/Put/seqid=0 2024-11-16T14:52:19,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768799664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768799666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768799667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768799669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768799669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742046_1222 (size=12001) 2024-11-16T14:52:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T14:52:19,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T14:52:19,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:19,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:19,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768799780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768799780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768799781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768799781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768799781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T14:52:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:19,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:19,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:19,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T14:52:19,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768799990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768799990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768799991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768799991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768799991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/34b47461cbd64364a095c46a9bf28aaf 2024-11-16T14:52:20,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T14:52:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/8a3eb24c4ac34216a025057536f6941b is 50, key is test_row_0/B:col10/1731768739632/Put/seqid=0 2024-11-16T14:52:20,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742047_1223 (size=12001) 2024-11-16T14:52:20,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T14:52:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:20,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T14:52:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768800300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768800301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768800301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:20,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768800301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768800301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T14:52:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/8a3eb24c4ac34216a025057536f6941b 2024-11-16T14:52:20,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T14:52:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/61fa3fc05a404b24a3ff76b9cb51370d is 50, key is test_row_0/C:col10/1731768739632/Put/seqid=0 2024-11-16T14:52:20,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742048_1224 (size=12001) 2024-11-16T14:52:20,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T14:52:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T14:52:20,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768800811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:20,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768800811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768800812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:20,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768800812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768800821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:20,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:20,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T14:52:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:20,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:20,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:20,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/61fa3fc05a404b24a3ff76b9cb51370d 2024-11-16T14:52:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/34b47461cbd64364a095c46a9bf28aaf as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/34b47461cbd64364a095c46a9bf28aaf 2024-11-16T14:52:20,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/34b47461cbd64364a095c46a9bf28aaf, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T14:52:20,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/8a3eb24c4ac34216a025057536f6941b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8a3eb24c4ac34216a025057536f6941b 2024-11-16T14:52:20,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8a3eb24c4ac34216a025057536f6941b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T14:52:20,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/61fa3fc05a404b24a3ff76b9cb51370d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/61fa3fc05a404b24a3ff76b9cb51370d 2024-11-16T14:52:20,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/61fa3fc05a404b24a3ff76b9cb51370d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T14:52:20,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0aa24bef827bbc1122cec8569b1d4491 in 1358ms, sequenceid=13, compaction requested=false 2024-11-16T14:52:20,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:21,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-16T14:52:21,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:21,003 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:52:21,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:21,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:21,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/025dc0e818bc4b26abd3b7b3002234d9 is 50, key is test_row_0/A:col10/1731768739667/Put/seqid=0 2024-11-16T14:52:21,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742049_1225 (size=12001) 2024-11-16T14:52:21,026 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/025dc0e818bc4b26abd3b7b3002234d9 2024-11-16T14:52:21,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/86ad886284a64ac0b204f306879d6228 is 50, key is test_row_0/B:col10/1731768739667/Put/seqid=0 2024-11-16T14:52:21,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742050_1226 (size=12001) 2024-11-16T14:52:21,069 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/86ad886284a64ac0b204f306879d6228 2024-11-16T14:52:21,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/0402ef749d6c493fb55804afdd66cb19 is 50, key is test_row_0/C:col10/1731768739667/Put/seqid=0 2024-11-16T14:52:21,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742051_1227 (size=12001) 2024-11-16T14:52:21,522 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/0402ef749d6c493fb55804afdd66cb19 2024-11-16T14:52:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/025dc0e818bc4b26abd3b7b3002234d9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/025dc0e818bc4b26abd3b7b3002234d9 2024-11-16T14:52:21,534 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/025dc0e818bc4b26abd3b7b3002234d9, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T14:52:21,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/86ad886284a64ac0b204f306879d6228 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/86ad886284a64ac0b204f306879d6228 2024-11-16T14:52:21,544 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/86ad886284a64ac0b204f306879d6228, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T14:52:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/0402ef749d6c493fb55804afdd66cb19 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/0402ef749d6c493fb55804afdd66cb19 2024-11-16T14:52:21,553 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/0402ef749d6c493fb55804afdd66cb19, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T14:52:21,554 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 0aa24bef827bbc1122cec8569b1d4491 in 551ms, sequenceid=33, compaction requested=false 2024-11-16T14:52:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:21,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-16T14:52:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-16T14:52:21,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-16T14:52:21,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9350 sec 2024-11-16T14:52:21,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.9420 sec 2024-11-16T14:52:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T14:52:21,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:21,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-16T14:52:21,774 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:21,775 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:21,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:52:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:21,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:52:21,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:21,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:21,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:21,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:21,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:21,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:21,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/52b92e90c784433eb478fc766cd1bdf2 is 50, key is test_row_0/A:col10/1731768741822/Put/seqid=0 2024-11-16T14:52:21,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742052_1228 (size=12001) 2024-11-16T14:52:21,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768741861, max timestamp=1731768741876Num uniques 859; 2024-11-16T14:52:21,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768741861, max timestamp=1731768741876Num uniques 859; 2024-11-16T14:52:21,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:21,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:52:21,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/52b92e90c784433eb478fc766cd1bdf2 2024-11-16T14:52:21,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768741861, max timestamp=1731768741876Num uniques 859; 2024-11-16T14:52:21,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:21,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/9432b89b059e4bbda7fc78e2b5475281 is 50, key is test_row_0/B:col10/1731768741822/Put/seqid=0 2024-11-16T14:52:21,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:52:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:21,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:21,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742053_1229 (size=12001) 2024-11-16T14:52:21,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/9432b89b059e4bbda7fc78e2b5475281 2024-11-16T14:52:21,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768801946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:21,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768801951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:21,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768801951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768801952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768801959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:21,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/f92e788d3d6f4da4a2245e2e9807f605 is 50, key is test_row_0/C:col10/1731768741822/Put/seqid=0 2024-11-16T14:52:22,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742054_1230 (size=12001) 2024-11-16T14:52:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768802070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768802071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768802071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768802071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768802071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:52:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:22,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:52:22,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:52:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:22,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:22,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768802280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768802291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768802291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768802291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768802292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:52:22,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:22,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:22,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:22,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:22,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:52:22,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/f92e788d3d6f4da4a2245e2e9807f605 2024-11-16T14:52:22,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/52b92e90c784433eb478fc766cd1bdf2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/52b92e90c784433eb478fc766cd1bdf2 2024-11-16T14:52:22,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/52b92e90c784433eb478fc766cd1bdf2, entries=150, sequenceid=44, filesize=11.7 K 2024-11-16T14:52:22,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/9432b89b059e4bbda7fc78e2b5475281 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/9432b89b059e4bbda7fc78e2b5475281 2024-11-16T14:52:22,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/9432b89b059e4bbda7fc78e2b5475281, entries=150, sequenceid=44, filesize=11.7 K 2024-11-16T14:52:22,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/f92e788d3d6f4da4a2245e2e9807f605 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f92e788d3d6f4da4a2245e2e9807f605 2024-11-16T14:52:22,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f92e788d3d6f4da4a2245e2e9807f605, entries=150, sequenceid=44, filesize=11.7 K 2024-11-16T14:52:22,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0aa24bef827bbc1122cec8569b1d4491 in 603ms, sequenceid=44, compaction requested=true 2024-11-16T14:52:22,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:22,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:22,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:22,439 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:22,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:22,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:22,439 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:22,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:22,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:22,440 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:22,440 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:22,440 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/A is initiating minor compaction (all files) 2024-11-16T14:52:22,440 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/B is initiating minor compaction (all files) 2024-11-16T14:52:22,440 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/A in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:22,440 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/B in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:22,440 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/34b47461cbd64364a095c46a9bf28aaf, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/025dc0e818bc4b26abd3b7b3002234d9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/52b92e90c784433eb478fc766cd1bdf2] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=35.2 K 2024-11-16T14:52:22,440 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8a3eb24c4ac34216a025057536f6941b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/86ad886284a64ac0b204f306879d6228, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/9432b89b059e4bbda7fc78e2b5475281] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=35.2 K 2024-11-16T14:52:22,440 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34b47461cbd64364a095c46a9bf28aaf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731768739632 2024-11-16T14:52:22,440 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a3eb24c4ac34216a025057536f6941b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731768739632 2024-11-16T14:52:22,440 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 025dc0e818bc4b26abd3b7b3002234d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731768739664 2024-11-16T14:52:22,440 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ad886284a64ac0b204f306879d6228, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731768739664 2024-11-16T14:52:22,440 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b92e90c784433eb478fc766cd1bdf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731768741822 2024-11-16T14:52:22,441 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9432b89b059e4bbda7fc78e2b5475281, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731768741822 2024-11-16T14:52:22,447 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#A#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:22,448 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#B#compaction#190 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:22,448 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/866e7a438824431691c103ded9de748a is 50, key is test_row_0/A:col10/1731768741822/Put/seqid=0 2024-11-16T14:52:22,448 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/663c87aa97b0478bbbf7cf63b33c4d24 is 50, key is test_row_0/B:col10/1731768741822/Put/seqid=0 2024-11-16T14:52:22,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742056_1232 (size=12104) 2024-11-16T14:52:22,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742055_1231 (size=12104) 2024-11-16T14:52:22,481 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/663c87aa97b0478bbbf7cf63b33c4d24 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/663c87aa97b0478bbbf7cf63b33c4d24 2024-11-16T14:52:22,487 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/B of 0aa24bef827bbc1122cec8569b1d4491 into 663c87aa97b0478bbbf7cf63b33c4d24(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:22,487 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:22,487 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/B, priority=13, startTime=1731768742439; duration=0sec 2024-11-16T14:52:22,487 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:22,487 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:B 2024-11-16T14:52:22,487 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:22,488 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:22,489 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/C is initiating minor compaction (all files) 2024-11-16T14:52:22,489 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/C in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:22,489 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/61fa3fc05a404b24a3ff76b9cb51370d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/0402ef749d6c493fb55804afdd66cb19, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f92e788d3d6f4da4a2245e2e9807f605] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=35.2 K 2024-11-16T14:52:22,490 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 61fa3fc05a404b24a3ff76b9cb51370d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731768739632 2024-11-16T14:52:22,490 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0402ef749d6c493fb55804afdd66cb19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731768739664 2024-11-16T14:52:22,491 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f92e788d3d6f4da4a2245e2e9807f605, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731768741822 2024-11-16T14:52:22,501 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#C#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:22,501 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/30aa8450bad24658a95360cf65f93079 is 50, key is test_row_0/C:col10/1731768741822/Put/seqid=0 2024-11-16T14:52:22,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742057_1233 (size=12104) 2024-11-16T14:52:22,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T14:52:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:22,545 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-16T14:52:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:22,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/1a4b6ed25c2943ca86d4477aedcf73e1 is 50, key is test_row_0/A:col10/1731768741891/Put/seqid=0 2024-11-16T14:52:22,595 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:52:22,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742058_1234 (size=12001) 2024-11-16T14:52:22,605 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/1a4b6ed25c2943ca86d4477aedcf73e1 2024-11-16T14:52:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:22,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/57e21ee468df4fefbc8c850c0e0a4254 is 50, key is test_row_0/B:col10/1731768741891/Put/seqid=0 2024-11-16T14:52:22,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768802628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768802624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768802628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768802634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768802636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742059_1235 (size=12001) 2024-11-16T14:52:22,673 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/57e21ee468df4fefbc8c850c0e0a4254 2024-11-16T14:52:22,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/f5d7e8f575ab43adba7a2c815cd3d70f is 50, key is test_row_0/C:col10/1731768741891/Put/seqid=0 2024-11-16T14:52:22,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742060_1236 (size=12001) 2024-11-16T14:52:22,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768802741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768802741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768802741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768802751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768802751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,885 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/866e7a438824431691c103ded9de748a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/866e7a438824431691c103ded9de748a 2024-11-16T14:52:22,911 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/A of 0aa24bef827bbc1122cec8569b1d4491 into 866e7a438824431691c103ded9de748a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:22,911 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:22,911 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/A, priority=13, startTime=1731768742438; duration=0sec 2024-11-16T14:52:22,911 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:22,911 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:A 2024-11-16T14:52:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:52:22,923 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/30aa8450bad24658a95360cf65f93079 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/30aa8450bad24658a95360cf65f93079 2024-11-16T14:52:22,928 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/C of 0aa24bef827bbc1122cec8569b1d4491 into 30aa8450bad24658a95360cf65f93079(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:22,928 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:22,928 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/C, priority=13, startTime=1731768742439; duration=0sec 2024-11-16T14:52:22,928 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:22,928 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:C 2024-11-16T14:52:22,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768802951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768802960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768802961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768802963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768802963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,145 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/f5d7e8f575ab43adba7a2c815cd3d70f 2024-11-16T14:52:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/1a4b6ed25c2943ca86d4477aedcf73e1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1a4b6ed25c2943ca86d4477aedcf73e1 2024-11-16T14:52:23,157 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1a4b6ed25c2943ca86d4477aedcf73e1, entries=150, sequenceid=80, filesize=11.7 K 2024-11-16T14:52:23,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/57e21ee468df4fefbc8c850c0e0a4254 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/57e21ee468df4fefbc8c850c0e0a4254 2024-11-16T14:52:23,163 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/57e21ee468df4fefbc8c850c0e0a4254, entries=150, sequenceid=80, filesize=11.7 K 2024-11-16T14:52:23,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/f5d7e8f575ab43adba7a2c815cd3d70f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f5d7e8f575ab43adba7a2c815cd3d70f 2024-11-16T14:52:23,175 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f5d7e8f575ab43adba7a2c815cd3d70f, entries=150, sequenceid=80, filesize=11.7 K 2024-11-16T14:52:23,177 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 0aa24bef827bbc1122cec8569b1d4491 in 632ms, sequenceid=80, compaction requested=false 2024-11-16T14:52:23,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:23,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:23,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-16T14:52:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-16T14:52:23,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-16T14:52:23,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-16T14:52:23,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4130 sec 2024-11-16T14:52:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:23,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:52:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:23,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/f882ccf9ac23459c94f1819fc9a34f09 is 50, key is test_row_0/A:col10/1731768742634/Put/seqid=0 2024-11-16T14:52:23,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742061_1237 (size=14341) 2024-11-16T14:52:23,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768803322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768803322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768803322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768803324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768803324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768803441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768803441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768803441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768803441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768803442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768803651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768803651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768803651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768803651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768803651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/f882ccf9ac23459c94f1819fc9a34f09 2024-11-16T14:52:23,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/2ad3b8eeeb8046e1b0a322e603207b3c is 50, key is test_row_0/B:col10/1731768742634/Put/seqid=0 2024-11-16T14:52:23,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742062_1238 (size=12001) 2024-11-16T14:52:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T14:52:23,930 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:23,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-16T14:52:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:52:23,933 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:23,934 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:23,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:23,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768803960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768803961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768803961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768803961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768803961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:52:24,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T14:52:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:24,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:24,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:24,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/2ad3b8eeeb8046e1b0a322e603207b3c 2024-11-16T14:52:24,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/e2311db667634a3f8db9f466b4a04515 is 50, key is test_row_0/C:col10/1731768742634/Put/seqid=0 2024-11-16T14:52:24,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742063_1239 (size=12001) 2024-11-16T14:52:24,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T14:52:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:24,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:24,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:52:24,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T14:52:24,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:24,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:24,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768804471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:24,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768804471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:24,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768804471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:24,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768804471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:24,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768804471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:24,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:24,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/e2311db667634a3f8db9f466b4a04515 2024-11-16T14:52:24,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/f882ccf9ac23459c94f1819fc9a34f09 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f882ccf9ac23459c94f1819fc9a34f09 2024-11-16T14:52:24,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f882ccf9ac23459c94f1819fc9a34f09, entries=200, sequenceid=94, filesize=14.0 K 2024-11-16T14:52:24,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/2ad3b8eeeb8046e1b0a322e603207b3c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2ad3b8eeeb8046e1b0a322e603207b3c 2024-11-16T14:52:24,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2ad3b8eeeb8046e1b0a322e603207b3c, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T14:52:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/e2311db667634a3f8db9f466b4a04515 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e2311db667634a3f8db9f466b4a04515 2024-11-16T14:52:24,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e2311db667634a3f8db9f466b4a04515, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T14:52:24,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0aa24bef827bbc1122cec8569b1d4491 in 1273ms, sequenceid=94, compaction requested=true 2024-11-16T14:52:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:24,537 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:24,537 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:24,539 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:24,539 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:24,539 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/A is initiating minor compaction (all files) 2024-11-16T14:52:24,539 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/B is initiating minor compaction (all files) 2024-11-16T14:52:24,539 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/A in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:24,539 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/B in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:24,539 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/663c87aa97b0478bbbf7cf63b33c4d24, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/57e21ee468df4fefbc8c850c0e0a4254, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2ad3b8eeeb8046e1b0a322e603207b3c] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=35.3 K 2024-11-16T14:52:24,539 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/866e7a438824431691c103ded9de748a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1a4b6ed25c2943ca86d4477aedcf73e1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f882ccf9ac23459c94f1819fc9a34f09] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=37.5 K 2024-11-16T14:52:24,540 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 866e7a438824431691c103ded9de748a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731768741822 2024-11-16T14:52:24,540 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 663c87aa97b0478bbbf7cf63b33c4d24, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731768741822 2024-11-16T14:52:24,540 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a4b6ed25c2943ca86d4477aedcf73e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731768741891 2024-11-16T14:52:24,540 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e21ee468df4fefbc8c850c0e0a4254, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731768741891 2024-11-16T14:52:24,540 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f882ccf9ac23459c94f1819fc9a34f09, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731768742611 2024-11-16T14:52:24,540 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ad3b8eeeb8046e1b0a322e603207b3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731768742618 2024-11-16T14:52:24,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T14:52:24,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:24,544 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:52:24,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/9efd082f5e124d088f1cd362075eba3d is 50, key is test_row_0/A:col10/1731768743288/Put/seqid=0 2024-11-16T14:52:24,556 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#B#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:24,556 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/88c8033d40a1485985e342ea30be281b is 50, key is test_row_0/B:col10/1731768742634/Put/seqid=0 2024-11-16T14:52:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:52:24,561 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#A#compaction#200 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:24,561 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/b68038045eb04676af56b6070e56b914 is 50, key is test_row_0/A:col10/1731768742634/Put/seqid=0 2024-11-16T14:52:24,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742064_1240 (size=12001) 2024-11-16T14:52:24,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742065_1241 (size=12207) 2024-11-16T14:52:24,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742066_1242 (size=12207) 2024-11-16T14:52:24,966 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/9efd082f5e124d088f1cd362075eba3d 2024-11-16T14:52:24,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/78401962a24d4722a9ebcb7d74c92cea is 50, key is test_row_0/B:col10/1731768743288/Put/seqid=0 2024-11-16T14:52:24,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742067_1243 (size=12001) 2024-11-16T14:52:24,988 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/88c8033d40a1485985e342ea30be281b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/88c8033d40a1485985e342ea30be281b 2024-11-16T14:52:24,991 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/b68038045eb04676af56b6070e56b914 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/b68038045eb04676af56b6070e56b914 2024-11-16T14:52:24,995 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/B of 0aa24bef827bbc1122cec8569b1d4491 into 88c8033d40a1485985e342ea30be281b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:24,995 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:24,995 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/B, priority=13, startTime=1731768744537; duration=0sec 2024-11-16T14:52:24,996 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:24,996 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:B 2024-11-16T14:52:24,996 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:25,000 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/A of 0aa24bef827bbc1122cec8569b1d4491 into b68038045eb04676af56b6070e56b914(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:25,000 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:25,000 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/A, priority=13, startTime=1731768744537; duration=0sec 2024-11-16T14:52:25,000 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:25,000 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:A 2024-11-16T14:52:25,000 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:25,001 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/C is initiating minor compaction (all files) 2024-11-16T14:52:25,001 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/C in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:25,001 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/30aa8450bad24658a95360cf65f93079, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f5d7e8f575ab43adba7a2c815cd3d70f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e2311db667634a3f8db9f466b4a04515] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=35.3 K 2024-11-16T14:52:25,001 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 30aa8450bad24658a95360cf65f93079, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731768741822 2024-11-16T14:52:25,002 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f5d7e8f575ab43adba7a2c815cd3d70f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731768741891 2024-11-16T14:52:25,002 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting e2311db667634a3f8db9f466b4a04515, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731768742618 2024-11-16T14:52:25,008 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:25,009 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/29c12a137e624439a9f8f519c8afda78 is 50, key is test_row_0/C:col10/1731768742634/Put/seqid=0 2024-11-16T14:52:25,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742068_1244 (size=12207) 2024-11-16T14:52:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:52:25,385 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/78401962a24d4722a9ebcb7d74c92cea 2024-11-16T14:52:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/c602113ffa90444daedb46e04a2983d6 is 50, key is test_row_0/C:col10/1731768743288/Put/seqid=0 2024-11-16T14:52:25,425 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/29c12a137e624439a9f8f519c8afda78 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/29c12a137e624439a9f8f519c8afda78 2024-11-16T14:52:25,430 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/C of 0aa24bef827bbc1122cec8569b1d4491 into 29c12a137e624439a9f8f519c8afda78(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:25,430 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:25,430 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/C, priority=13, startTime=1731768744537; duration=0sec 2024-11-16T14:52:25,430 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:25,431 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:C 2024-11-16T14:52:25,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742069_1245 (size=12001) 2024-11-16T14:52:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:25,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:25,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768805490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768805491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768805491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,496 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 0aa24bef827bbc1122cec8569b1d4491 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768805496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768805496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768805600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768805600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768805610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768805610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768805811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768805811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768805820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:25,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768805820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:25,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:25,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:25,840 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/c602113ffa90444daedb46e04a2983d6 2024-11-16T14:52:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/9efd082f5e124d088f1cd362075eba3d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/9efd082f5e124d088f1cd362075eba3d 2024-11-16T14:52:25,847 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/9efd082f5e124d088f1cd362075eba3d, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T14:52:25,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/78401962a24d4722a9ebcb7d74c92cea as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/78401962a24d4722a9ebcb7d74c92cea 2024-11-16T14:52:25,853 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/78401962a24d4722a9ebcb7d74c92cea, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T14:52:25,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/c602113ffa90444daedb46e04a2983d6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c602113ffa90444daedb46e04a2983d6 2024-11-16T14:52:25,860 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c602113ffa90444daedb46e04a2983d6, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T14:52:25,861 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 0aa24bef827bbc1122cec8569b1d4491 in 1317ms, sequenceid=114, compaction requested=false 2024-11-16T14:52:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-16T14:52:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-16T14:52:25,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-16T14:52:25,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9290 sec 2024-11-16T14:52:25,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.9350 sec 2024-11-16T14:52:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T14:52:26,080 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:26,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-16T14:52:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:52:26,082 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:26,083 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:26,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:26,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T14:52:26,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:26,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/6620e262fed247ba96582f50bd76b5e0 is 50, key is test_row_0/A:col10/1731768746122/Put/seqid=0 2024-11-16T14:52:26,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742070_1246 (size=14491) 2024-11-16T14:52:26,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/6620e262fed247ba96582f50bd76b5e0 2024-11-16T14:52:26,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/658c8cdf3a9546f5ae1a00f467fbe6c0 is 50, key is test_row_0/B:col10/1731768746122/Put/seqid=0 2024-11-16T14:52:26,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742071_1247 (size=12151) 2024-11-16T14:52:26,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768806172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768806172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768806173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768806179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:52:26,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:52:26,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:26,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:26,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:26,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:26,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:26,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768806280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768806291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768806291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768806291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:52:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:26,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:52:26,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768806491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768806500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768806500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768806500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:52:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:26,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:26,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/658c8cdf3a9546f5ae1a00f467fbe6c0 2024-11-16T14:52:26,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/4f57d0b05f0d4e38b75e6846e8ceddb0 is 50, key is test_row_0/C:col10/1731768746122/Put/seqid=0 2024-11-16T14:52:26,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742072_1248 (size=12151) 2024-11-16T14:52:26,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/4f57d0b05f0d4e38b75e6846e8ceddb0 2024-11-16T14:52:26,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/6620e262fed247ba96582f50bd76b5e0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6620e262fed247ba96582f50bd76b5e0 2024-11-16T14:52:26,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6620e262fed247ba96582f50bd76b5e0, entries=200, sequenceid=132, filesize=14.2 K 2024-11-16T14:52:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/658c8cdf3a9546f5ae1a00f467fbe6c0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/658c8cdf3a9546f5ae1a00f467fbe6c0 2024-11-16T14:52:26,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/658c8cdf3a9546f5ae1a00f467fbe6c0, entries=150, sequenceid=132, filesize=11.9 K 2024-11-16T14:52:26,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/4f57d0b05f0d4e38b75e6846e8ceddb0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/4f57d0b05f0d4e38b75e6846e8ceddb0 2024-11-16T14:52:26,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/4f57d0b05f0d4e38b75e6846e8ceddb0, entries=150, sequenceid=132, filesize=11.9 K 2024-11-16T14:52:26,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0aa24bef827bbc1122cec8569b1d4491 in 457ms, sequenceid=132, compaction requested=true 2024-11-16T14:52:26,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:26,581 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:26,581 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:26,582 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:26,582 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/B is initiating minor compaction (all files) 2024-11-16T14:52:26,582 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/B in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:26,582 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/88c8033d40a1485985e342ea30be281b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/78401962a24d4722a9ebcb7d74c92cea, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/658c8cdf3a9546f5ae1a00f467fbe6c0] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=35.5 K 2024-11-16T14:52:26,583 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:26,583 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 88c8033d40a1485985e342ea30be281b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731768742618 2024-11-16T14:52:26,583 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/A is initiating minor compaction (all files) 2024-11-16T14:52:26,583 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/A in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:26,583 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/b68038045eb04676af56b6070e56b914, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/9efd082f5e124d088f1cd362075eba3d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6620e262fed247ba96582f50bd76b5e0] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=37.8 K 2024-11-16T14:52:26,583 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 78401962a24d4722a9ebcb7d74c92cea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731768743288 2024-11-16T14:52:26,584 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b68038045eb04676af56b6070e56b914, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731768742618 2024-11-16T14:52:26,584 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 658c8cdf3a9546f5ae1a00f467fbe6c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731768745494 2024-11-16T14:52:26,584 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9efd082f5e124d088f1cd362075eba3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731768743288 2024-11-16T14:52:26,584 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6620e262fed247ba96582f50bd76b5e0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731768745494 2024-11-16T14:52:26,592 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:26,592 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/2aa3e8f757c94ce18692e3143aa1f7f5 is 50, key is test_row_0/B:col10/1731768746122/Put/seqid=0 2024-11-16T14:52:26,595 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#A#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:26,595 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/1d0c26c54e7546bdb4f4587f02e677a1 is 50, key is test_row_0/A:col10/1731768746122/Put/seqid=0 2024-11-16T14:52:26,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742073_1249 (size=12459) 2024-11-16T14:52:26,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742074_1250 (size=12459) 2024-11-16T14:52:26,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T14:52:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:26,694 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T14:52:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:26,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:26,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/20260297f1974162812f7e1b47af03eb is 50, key is test_row_0/A:col10/1731768746146/Put/seqid=0 2024-11-16T14:52:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:52:26,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742075_1251 (size=12151) 2024-11-16T14:52:26,722 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/20260297f1974162812f7e1b47af03eb 2024-11-16T14:52:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/e8eae55971504f828a8431de04240bee is 50, key is test_row_0/B:col10/1731768746146/Put/seqid=0 2024-11-16T14:52:26,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742076_1252 (size=12151) 2024-11-16T14:52:26,758 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/e8eae55971504f828a8431de04240bee 2024-11-16T14:52:26,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/1c638c6f73444d3eb701c9a4fe44fd14 is 50, key is test_row_0/C:col10/1731768746146/Put/seqid=0 2024-11-16T14:52:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:26,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:26,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742077_1253 (size=12151) 2024-11-16T14:52:26,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768806851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768806852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768806852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768806858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768806961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768806961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768806970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:26,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768806970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,010 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/2aa3e8f757c94ce18692e3143aa1f7f5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2aa3e8f757c94ce18692e3143aa1f7f5 2024-11-16T14:52:27,016 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/B of 0aa24bef827bbc1122cec8569b1d4491 into 2aa3e8f757c94ce18692e3143aa1f7f5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:27,016 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:27,016 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/B, priority=13, startTime=1731768746581; duration=0sec 2024-11-16T14:52:27,017 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:27,017 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:B 2024-11-16T14:52:27,017 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:27,018 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:27,018 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/C is initiating minor compaction (all files) 2024-11-16T14:52:27,018 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/C in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:27,019 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/29c12a137e624439a9f8f519c8afda78, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c602113ffa90444daedb46e04a2983d6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/4f57d0b05f0d4e38b75e6846e8ceddb0] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=35.5 K 2024-11-16T14:52:27,021 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 29c12a137e624439a9f8f519c8afda78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731768742618 2024-11-16T14:52:27,021 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting c602113ffa90444daedb46e04a2983d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731768743288 2024-11-16T14:52:27,022 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f57d0b05f0d4e38b75e6846e8ceddb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731768745494 2024-11-16T14:52:27,029 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/1d0c26c54e7546bdb4f4587f02e677a1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1d0c26c54e7546bdb4f4587f02e677a1 2024-11-16T14:52:27,033 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/A of 0aa24bef827bbc1122cec8569b1d4491 into 1d0c26c54e7546bdb4f4587f02e677a1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:27,033 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:27,033 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/A, priority=13, startTime=1731768746580; duration=0sec 2024-11-16T14:52:27,033 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:27,033 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:A 2024-11-16T14:52:27,037 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:27,037 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/400757b47500421e87295e9d1abbd89e is 50, key is test_row_0/C:col10/1731768746122/Put/seqid=0 2024-11-16T14:52:27,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742078_1254 (size=12459) 2024-11-16T14:52:27,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768807171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768807171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768807180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768807187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,206 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/1c638c6f73444d3eb701c9a4fe44fd14 2024-11-16T14:52:27,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/20260297f1974162812f7e1b47af03eb as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/20260297f1974162812f7e1b47af03eb 2024-11-16T14:52:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:52:27,221 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/20260297f1974162812f7e1b47af03eb, entries=150, sequenceid=149, filesize=11.9 K 2024-11-16T14:52:27,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/e8eae55971504f828a8431de04240bee as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e8eae55971504f828a8431de04240bee 2024-11-16T14:52:27,228 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e8eae55971504f828a8431de04240bee, entries=150, sequenceid=149, filesize=11.9 K 2024-11-16T14:52:27,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/1c638c6f73444d3eb701c9a4fe44fd14 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1c638c6f73444d3eb701c9a4fe44fd14 2024-11-16T14:52:27,235 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1c638c6f73444d3eb701c9a4fe44fd14, entries=150, sequenceid=149, filesize=11.9 K 2024-11-16T14:52:27,236 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0aa24bef827bbc1122cec8569b1d4491 in 542ms, sequenceid=149, compaction requested=false 2024-11-16T14:52:27,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:27,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:27,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-16T14:52:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-16T14:52:27,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-16T14:52:27,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1550 sec 2024-11-16T14:52:27,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1670 sec 2024-11-16T14:52:27,457 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/400757b47500421e87295e9d1abbd89e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/400757b47500421e87295e9d1abbd89e 2024-11-16T14:52:27,461 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/C of 0aa24bef827bbc1122cec8569b1d4491 into 400757b47500421e87295e9d1abbd89e(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:27,461 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:27,461 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/C, priority=13, startTime=1731768746581; duration=0sec 2024-11-16T14:52:27,461 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:27,461 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:C 2024-11-16T14:52:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:27,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T14:52:27,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:27,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:27,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:27,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:27,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:27,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:27,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/87dc311d1bd64edebdd80517dd1fac25 is 50, key is test_row_0/A:col10/1731768746857/Put/seqid=0 2024-11-16T14:52:27,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742079_1255 (size=14541) 2024-11-16T14:52:27,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768807506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768807507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768807507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768807511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768807511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,515 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 0aa24bef827bbc1122cec8569b1d4491 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768807621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768807621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768807621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768807621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768807831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768807831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768807831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768807831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:27,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/87dc311d1bd64edebdd80517dd1fac25 2024-11-16T14:52:27,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/63e78c94c2234f8283fcfb3fb89384f3 is 50, key is test_row_0/B:col10/1731768746857/Put/seqid=0 2024-11-16T14:52:27,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742080_1256 (size=12151) 2024-11-16T14:52:27,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/63e78c94c2234f8283fcfb3fb89384f3 2024-11-16T14:52:27,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/7e72509d5da1499f80c944655e927551 is 50, key is test_row_0/C:col10/1731768746857/Put/seqid=0 2024-11-16T14:52:27,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742081_1257 (size=12151) 2024-11-16T14:52:28,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768808141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768808141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768808141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768808141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T14:52:28,231 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:28,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-16T14:52:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:52:28,233 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:28,233 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:28,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:52:28,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/7e72509d5da1499f80c944655e927551 2024-11-16T14:52:28,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/87dc311d1bd64edebdd80517dd1fac25 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/87dc311d1bd64edebdd80517dd1fac25 2024-11-16T14:52:28,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/87dc311d1bd64edebdd80517dd1fac25, entries=200, sequenceid=169, filesize=14.2 K 2024-11-16T14:52:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/63e78c94c2234f8283fcfb3fb89384f3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/63e78c94c2234f8283fcfb3fb89384f3 2024-11-16T14:52:28,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/63e78c94c2234f8283fcfb3fb89384f3, entries=150, sequenceid=169, filesize=11.9 K 2024-11-16T14:52:28,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/7e72509d5da1499f80c944655e927551 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7e72509d5da1499f80c944655e927551 2024-11-16T14:52:28,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:52:28,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:28,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:28,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:28,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:28,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7e72509d5da1499f80c944655e927551, entries=150, sequenceid=169, filesize=11.9 K 2024-11-16T14:52:28,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0aa24bef827bbc1122cec8569b1d4491 in 920ms, sequenceid=169, compaction requested=true 2024-11-16T14:52:28,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:28,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:28,403 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:28,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:28,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:28,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:28,403 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:28,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:28,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:28,404 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:28,404 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:28,404 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/B is initiating minor compaction (all files) 2024-11-16T14:52:28,404 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/A is initiating minor compaction (all files) 2024-11-16T14:52:28,404 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/B in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:28,404 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/A in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:28,404 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1d0c26c54e7546bdb4f4587f02e677a1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/20260297f1974162812f7e1b47af03eb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/87dc311d1bd64edebdd80517dd1fac25] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=38.2 K 2024-11-16T14:52:28,404 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2aa3e8f757c94ce18692e3143aa1f7f5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e8eae55971504f828a8431de04240bee, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/63e78c94c2234f8283fcfb3fb89384f3] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=35.9 K 2024-11-16T14:52:28,405 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa3e8f757c94ce18692e3143aa1f7f5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731768745494 2024-11-16T14:52:28,405 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d0c26c54e7546bdb4f4587f02e677a1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731768745494 2024-11-16T14:52:28,405 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20260297f1974162812f7e1b47af03eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731768746146 2024-11-16T14:52:28,405 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting e8eae55971504f828a8431de04240bee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731768746146 2024-11-16T14:52:28,405 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 63e78c94c2234f8283fcfb3fb89384f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731768746816 2024-11-16T14:52:28,405 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87dc311d1bd64edebdd80517dd1fac25, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731768746816 2024-11-16T14:52:28,413 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#B#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:28,413 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/11863d754dfe4b46a787d4308f4335f2 is 50, key is test_row_0/B:col10/1731768746857/Put/seqid=0 2024-11-16T14:52:28,416 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:28,416 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/860d5636d0974a15898c9fb844114a12 is 50, key is test_row_0/A:col10/1731768746857/Put/seqid=0 2024-11-16T14:52:28,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742082_1258 (size=12561) 2024-11-16T14:52:28,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742083_1259 (size=12561) 2024-11-16T14:52:28,434 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/860d5636d0974a15898c9fb844114a12 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/860d5636d0974a15898c9fb844114a12 2024-11-16T14:52:28,439 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/A of 0aa24bef827bbc1122cec8569b1d4491 into 860d5636d0974a15898c9fb844114a12(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:28,439 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:28,439 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/A, priority=13, startTime=1731768748403; duration=0sec 2024-11-16T14:52:28,439 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:28,439 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:A 2024-11-16T14:52:28,439 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:28,440 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:28,440 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/C is initiating minor compaction (all files) 2024-11-16T14:52:28,440 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/C in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:28,440 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/400757b47500421e87295e9d1abbd89e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1c638c6f73444d3eb701c9a4fe44fd14, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7e72509d5da1499f80c944655e927551] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=35.9 K 2024-11-16T14:52:28,441 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 400757b47500421e87295e9d1abbd89e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731768745494 2024-11-16T14:52:28,441 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c638c6f73444d3eb701c9a4fe44fd14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731768746146 2024-11-16T14:52:28,441 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e72509d5da1499f80c944655e927551, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731768746816 2024-11-16T14:52:28,447 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:28,447 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/e7953bb07304481d903d8717a9f1a9a9 is 50, key is test_row_0/C:col10/1731768746857/Put/seqid=0 2024-11-16T14:52:28,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742084_1260 (size=12561) 2024-11-16T14:52:28,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T14:52:28,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:28,541 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T14:52:28,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:28,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:28,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:28,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:28,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:28,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:28,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/c693a314614b45bea8436c693a225181 is 50, key is test_row_0/A:col10/1731768747505/Put/seqid=0 2024-11-16T14:52:28,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742085_1261 (size=12151) 2024-11-16T14:52:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:52:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:28,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:28,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768808667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768808667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768808674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768808675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768808780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768808781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768808791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768808791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,831 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/11863d754dfe4b46a787d4308f4335f2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/11863d754dfe4b46a787d4308f4335f2 2024-11-16T14:52:28,835 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/B of 0aa24bef827bbc1122cec8569b1d4491 into 11863d754dfe4b46a787d4308f4335f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:28,835 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:28,835 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/B, priority=13, startTime=1731768748403; duration=0sec 2024-11-16T14:52:28,835 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:28,835 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:B 2024-11-16T14:52:28,855 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/e7953bb07304481d903d8717a9f1a9a9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e7953bb07304481d903d8717a9f1a9a9 2024-11-16T14:52:28,860 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/C of 0aa24bef827bbc1122cec8569b1d4491 into e7953bb07304481d903d8717a9f1a9a9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:28,860 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:28,860 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/C, priority=13, startTime=1731768748403; duration=0sec 2024-11-16T14:52:28,860 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:28,860 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:C 2024-11-16T14:52:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:52:28,951 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/c693a314614b45bea8436c693a225181 2024-11-16T14:52:28,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/afe81161d6de4d94a35de49fbf8a3195 is 50, key is test_row_0/B:col10/1731768747505/Put/seqid=0 2024-11-16T14:52:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768808990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768808990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:28,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742086_1262 (size=12151) 2024-11-16T14:52:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768809001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:29,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768809001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768809301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768809301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768809310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:29,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768809310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:52:29,395 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/afe81161d6de4d94a35de49fbf8a3195 2024-11-16T14:52:29,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/5557f5bd5d504fc491c1be20fd1658e9 is 50, key is test_row_0/C:col10/1731768747505/Put/seqid=0 2024-11-16T14:52:29,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742087_1263 (size=12151) 2024-11-16T14:52:29,809 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/5557f5bd5d504fc491c1be20fd1658e9 2024-11-16T14:52:29,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768809810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:29,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768809810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768809820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768809821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:29,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/c693a314614b45bea8436c693a225181 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/c693a314614b45bea8436c693a225181 2024-11-16T14:52:29,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:29,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:29,836 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/c693a314614b45bea8436c693a225181, entries=150, sequenceid=185, filesize=11.9 K 2024-11-16T14:52:29,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/afe81161d6de4d94a35de49fbf8a3195 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/afe81161d6de4d94a35de49fbf8a3195 2024-11-16T14:52:29,853 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/afe81161d6de4d94a35de49fbf8a3195, entries=150, sequenceid=185, filesize=11.9 K 2024-11-16T14:52:29,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/5557f5bd5d504fc491c1be20fd1658e9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5557f5bd5d504fc491c1be20fd1658e9 2024-11-16T14:52:29,860 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5557f5bd5d504fc491c1be20fd1658e9, entries=150, sequenceid=185, filesize=11.9 K 2024-11-16T14:52:29,861 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0aa24bef827bbc1122cec8569b1d4491 in 1320ms, sequenceid=185, compaction requested=false 2024-11-16T14:52:29,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:29,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:29,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-16T14:52:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-16T14:52:29,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-16T14:52:29,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6310 sec 2024-11-16T14:52:29,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6360 sec 2024-11-16T14:52:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T14:52:30,381 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:30,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-16T14:52:30,386 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T14:52:30,388 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:30,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T14:52:30,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T14:52:30,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:30,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T14:52:30,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:30,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:30,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:30,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:30,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/42847b07edb74148a80d77fccbed533a is 50, key is test_row_0/A:col10/1731768748671/Put/seqid=0 2024-11-16T14:52:30,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742088_1264 (size=12151) 2024-11-16T14:52:30,596 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/42847b07edb74148a80d77fccbed533a 2024-11-16T14:52:30,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/e848b25c68aa4060910c48e950452eac is 50, key is test_row_0/B:col10/1731768748671/Put/seqid=0 2024-11-16T14:52:30,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742089_1265 (size=12151) 2024-11-16T14:52:30,665 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/e848b25c68aa4060910c48e950452eac 2024-11-16T14:52:30,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/1a19db89278b497184636b526bb7e0cb is 50, key is test_row_0/C:col10/1731768748671/Put/seqid=0 2024-11-16T14:52:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T14:52:30,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742090_1266 (size=12151) 2024-11-16T14:52:30,721 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/1a19db89278b497184636b526bb7e0cb 2024-11-16T14:52:30,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/42847b07edb74148a80d77fccbed533a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/42847b07edb74148a80d77fccbed533a 2024-11-16T14:52:30,738 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/42847b07edb74148a80d77fccbed533a, entries=150, sequenceid=204, filesize=11.9 K 2024-11-16T14:52:30,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/e848b25c68aa4060910c48e950452eac as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e848b25c68aa4060910c48e950452eac 2024-11-16T14:52:30,756 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e848b25c68aa4060910c48e950452eac, entries=150, sequenceid=204, filesize=11.9 K 2024-11-16T14:52:30,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/1a19db89278b497184636b526bb7e0cb as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1a19db89278b497184636b526bb7e0cb 2024-11-16T14:52:30,768 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1a19db89278b497184636b526bb7e0cb, entries=150, sequenceid=204, filesize=11.9 K 2024-11-16T14:52:30,769 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 0aa24bef827bbc1122cec8569b1d4491 in 223ms, sequenceid=204, compaction requested=true 2024-11-16T14:52:30,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:30,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:30,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-16T14:52:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-16T14:52:30,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-16T14:52:30,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 383 msec 2024-11-16T14:52:30,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 394 msec 2024-11-16T14:52:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:30,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:52:30,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:30,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:30,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:30,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:30,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:30,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:30,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/258b62901ecd4016ae7652660f25f806 is 50, key is test_row_0/A:col10/1731768750831/Put/seqid=0 2024-11-16T14:52:30,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742091_1267 (size=12151) 2024-11-16T14:52:30,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768750876, max timestamp=1731768750883Num uniques 859; 2024-11-16T14:52:30,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:52:30,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768750876, max timestamp=1731768750883Num uniques 859; 2024-11-16T14:52:30,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:52:30,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768750876, max timestamp=1731768750883Num uniques 859; 2024-11-16T14:52:30,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:52:30,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:30,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:30,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768810923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768810924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768810928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:30,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768810931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T14:52:31,011 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:31,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-16T14:52:31,014 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:31,015 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:31,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:52:31,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768811040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768811040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768811040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768811041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:52:31,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:52:31,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:31,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768811251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768811251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768811251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768811251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/258b62901ecd4016ae7652660f25f806 2024-11-16T14:52:31,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/09f02e2aaeff42acaf4d2c92c9a03ede is 50, key is test_row_0/B:col10/1731768750831/Put/seqid=0 2024-11-16T14:52:31,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:52:31,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:31,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742092_1268 (size=12151) 2024-11-16T14:52:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:52:31,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:52:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731768811532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,534 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 0aa24bef827bbc1122cec8569b1d4491 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768811561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768811561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768811561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768811570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:31,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:52:31,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:31,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:52:31,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/09f02e2aaeff42acaf4d2c92c9a03ede 2024-11-16T14:52:31,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/b9ebe63117a64969b1757d9d0848dc94 is 50, key is test_row_0/C:col10/1731768750831/Put/seqid=0 2024-11-16T14:52:31,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742093_1269 (size=12151) 2024-11-16T14:52:31,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:52:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:52:31,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:31,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:31,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:31,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:32,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768812080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:32,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:32,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768812080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:32,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768812080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:32,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:52:32,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:32,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:32,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:32,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:32,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:32,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768812091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:52:32,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/b9ebe63117a64969b1757d9d0848dc94 2024-11-16T14:52:32,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/258b62901ecd4016ae7652660f25f806 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/258b62901ecd4016ae7652660f25f806 2024-11-16T14:52:32,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/258b62901ecd4016ae7652660f25f806, entries=150, sequenceid=215, filesize=11.9 K 2024-11-16T14:52:32,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/09f02e2aaeff42acaf4d2c92c9a03ede as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/09f02e2aaeff42acaf4d2c92c9a03ede 2024-11-16T14:52:32,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/09f02e2aaeff42acaf4d2c92c9a03ede, entries=150, sequenceid=215, filesize=11.9 K 2024-11-16T14:52:32,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/b9ebe63117a64969b1757d9d0848dc94 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b9ebe63117a64969b1757d9d0848dc94 2024-11-16T14:52:32,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b9ebe63117a64969b1757d9d0848dc94, entries=150, sequenceid=215, filesize=11.9 K 2024-11-16T14:52:32,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0aa24bef827bbc1122cec8569b1d4491 in 1364ms, sequenceid=215, compaction requested=true 2024-11-16T14:52:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:32,205 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:32,205 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:32,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:32,207 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:32,207 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/B is initiating minor compaction (all files) 2024-11-16T14:52:32,207 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/B in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:32,207 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/11863d754dfe4b46a787d4308f4335f2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/afe81161d6de4d94a35de49fbf8a3195, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e848b25c68aa4060910c48e950452eac, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/09f02e2aaeff42acaf4d2c92c9a03ede] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=47.9 K 2024-11-16T14:52:32,207 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:32,207 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/A is initiating minor compaction (all files) 2024-11-16T14:52:32,207 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/A in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:32,207 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/860d5636d0974a15898c9fb844114a12, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/c693a314614b45bea8436c693a225181, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/42847b07edb74148a80d77fccbed533a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/258b62901ecd4016ae7652660f25f806] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=47.9 K 2024-11-16T14:52:32,209 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 860d5636d0974a15898c9fb844114a12, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731768746816 2024-11-16T14:52:32,209 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 11863d754dfe4b46a787d4308f4335f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731768746816 2024-11-16T14:52:32,209 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c693a314614b45bea8436c693a225181, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731768747503 2024-11-16T14:52:32,209 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting afe81161d6de4d94a35de49fbf8a3195, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731768747503 2024-11-16T14:52:32,209 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42847b07edb74148a80d77fccbed533a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731768748665 2024-11-16T14:52:32,210 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 258b62901ecd4016ae7652660f25f806, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731768750830 2024-11-16T14:52:32,210 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting e848b25c68aa4060910c48e950452eac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731768748665 2024-11-16T14:52:32,211 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 09f02e2aaeff42acaf4d2c92c9a03ede, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731768750830 2024-11-16T14:52:32,223 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:32,224 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/83547d7d7bd649e19d51134005024da3 is 50, key is test_row_0/B:col10/1731768750831/Put/seqid=0 2024-11-16T14:52:32,227 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:32,228 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/cb4a363e5dae4f2eb8d69cd17618827f is 50, key is test_row_0/A:col10/1731768750831/Put/seqid=0 2024-11-16T14:52:32,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T14:52:32,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:32,249 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T14:52:32,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:32,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:32,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:32,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:32,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:32,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:32,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742094_1270 (size=12697) 2024-11-16T14:52:32,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742095_1271 (size=12697) 2024-11-16T14:52:32,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/ca52d44b7cbf47b890b89895ffe3d544 is 50, key is test_row_0/A:col10/1731768750927/Put/seqid=0 2024-11-16T14:52:32,294 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/cb4a363e5dae4f2eb8d69cd17618827f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/cb4a363e5dae4f2eb8d69cd17618827f 2024-11-16T14:52:32,300 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/A of 0aa24bef827bbc1122cec8569b1d4491 into cb4a363e5dae4f2eb8d69cd17618827f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:32,300 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:32,300 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/A, priority=12, startTime=1731768752205; duration=0sec 2024-11-16T14:52:32,300 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:32,300 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:A 2024-11-16T14:52:32,300 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:32,315 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:32,316 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/C is initiating minor compaction (all files) 2024-11-16T14:52:32,316 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/C in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:32,316 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e7953bb07304481d903d8717a9f1a9a9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5557f5bd5d504fc491c1be20fd1658e9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1a19db89278b497184636b526bb7e0cb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b9ebe63117a64969b1757d9d0848dc94] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=47.9 K 2024-11-16T14:52:32,316 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7953bb07304481d903d8717a9f1a9a9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731768746816 2024-11-16T14:52:32,317 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5557f5bd5d504fc491c1be20fd1658e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731768747503 2024-11-16T14:52:32,317 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a19db89278b497184636b526bb7e0cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731768748665 2024-11-16T14:52:32,317 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9ebe63117a64969b1757d9d0848dc94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731768750830 2024-11-16T14:52:32,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742096_1272 (size=12151) 2024-11-16T14:52:32,349 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#C#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:32,350 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/7ae9b81771e34aee88fdd6175a1a7c3d is 50, key is test_row_0/C:col10/1731768750831/Put/seqid=0 2024-11-16T14:52:32,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742097_1273 (size=12697) 2024-11-16T14:52:32,387 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/7ae9b81771e34aee88fdd6175a1a7c3d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7ae9b81771e34aee88fdd6175a1a7c3d 2024-11-16T14:52:32,391 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/C of 0aa24bef827bbc1122cec8569b1d4491 into 7ae9b81771e34aee88fdd6175a1a7c3d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:32,391 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:32,391 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/C, priority=12, startTime=1731768752205; duration=0sec 2024-11-16T14:52:32,391 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:32,391 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:C 2024-11-16T14:52:32,665 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/83547d7d7bd649e19d51134005024da3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/83547d7d7bd649e19d51134005024da3 2024-11-16T14:52:32,671 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/B of 0aa24bef827bbc1122cec8569b1d4491 into 83547d7d7bd649e19d51134005024da3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:32,671 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:32,671 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/B, priority=12, startTime=1731768752205; duration=0sec 2024-11-16T14:52:32,671 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:32,671 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:B 2024-11-16T14:52:32,735 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/ca52d44b7cbf47b890b89895ffe3d544 2024-11-16T14:52:32,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/959c9c87e575404a8c918d68a80f6bdd is 50, key is test_row_0/B:col10/1731768750927/Put/seqid=0 2024-11-16T14:52:32,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742098_1274 (size=12151) 2024-11-16T14:52:32,788 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/959c9c87e575404a8c918d68a80f6bdd 2024-11-16T14:52:32,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/b2cfd753900f40cb849b43ed52a0d2d1 is 50, key is test_row_0/C:col10/1731768750927/Put/seqid=0 2024-11-16T14:52:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742099_1275 (size=12151) 2024-11-16T14:52:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:33,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:33,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768813110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768813111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,115 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in 0aa24bef827bbc1122cec8569b1d4491 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768813112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768813113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:52:33,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768813220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768813220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768813220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,256 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/b2cfd753900f40cb849b43ed52a0d2d1 2024-11-16T14:52:33,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/ca52d44b7cbf47b890b89895ffe3d544 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/ca52d44b7cbf47b890b89895ffe3d544 2024-11-16T14:52:33,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/ca52d44b7cbf47b890b89895ffe3d544, entries=150, sequenceid=251, filesize=11.9 K 2024-11-16T14:52:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/959c9c87e575404a8c918d68a80f6bdd as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/959c9c87e575404a8c918d68a80f6bdd 2024-11-16T14:52:33,268 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/959c9c87e575404a8c918d68a80f6bdd, entries=150, sequenceid=251, filesize=11.9 K 2024-11-16T14:52:33,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/b2cfd753900f40cb849b43ed52a0d2d1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b2cfd753900f40cb849b43ed52a0d2d1 2024-11-16T14:52:33,272 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b2cfd753900f40cb849b43ed52a0d2d1, entries=150, sequenceid=251, filesize=11.9 K 2024-11-16T14:52:33,273 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 0aa24bef827bbc1122cec8569b1d4491 in 1024ms, sequenceid=251, compaction requested=false 2024-11-16T14:52:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-16T14:52:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-16T14:52:33,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-16T14:52:33,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2600 sec 2024-11-16T14:52:33,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.2650 sec 2024-11-16T14:52:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:33,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:52:33,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:33,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:33,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:33,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:33,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:33,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:33,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/0c09c504f1404beaacbebbd52cdf93a8 is 50, key is test_row_0/A:col10/1731768753432/Put/seqid=0 2024-11-16T14:52:33,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742100_1276 (size=12301) 2024-11-16T14:52:33,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768813456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768813458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768813459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768813570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768813570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768813570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768813781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768813781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768813781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:33,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/0c09c504f1404beaacbebbd52cdf93a8 2024-11-16T14:52:33,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/aa65ba8ddae14eb5a9ded7c787d567b5 is 50, key is test_row_0/B:col10/1731768753432/Put/seqid=0 2024-11-16T14:52:33,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742101_1277 (size=12301) 2024-11-16T14:52:34,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768814090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768814090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768814090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:34,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/aa65ba8ddae14eb5a9ded7c787d567b5 2024-11-16T14:52:34,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/b6922e3ed468410896d55477f2dad08e is 50, key is test_row_0/C:col10/1731768753432/Put/seqid=0 2024-11-16T14:52:34,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742102_1278 (size=12301) 2024-11-16T14:52:34,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768814600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:34,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768814600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768814601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:34,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/b6922e3ed468410896d55477f2dad08e 2024-11-16T14:52:34,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/0c09c504f1404beaacbebbd52cdf93a8 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/0c09c504f1404beaacbebbd52cdf93a8 2024-11-16T14:52:34,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/0c09c504f1404beaacbebbd52cdf93a8, entries=150, sequenceid=267, filesize=12.0 K 2024-11-16T14:52:34,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/aa65ba8ddae14eb5a9ded7c787d567b5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/aa65ba8ddae14eb5a9ded7c787d567b5 2024-11-16T14:52:34,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/aa65ba8ddae14eb5a9ded7c787d567b5, entries=150, sequenceid=267, filesize=12.0 K 2024-11-16T14:52:34,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/b6922e3ed468410896d55477f2dad08e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b6922e3ed468410896d55477f2dad08e 2024-11-16T14:52:34,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b6922e3ed468410896d55477f2dad08e, entries=150, sequenceid=267, filesize=12.0 K 2024-11-16T14:52:34,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0aa24bef827bbc1122cec8569b1d4491 in 1288ms, sequenceid=267, compaction requested=true 2024-11-16T14:52:34,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:34,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:34,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:34,720 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:34,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:34,720 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:34,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:34,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:34,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:34,721 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:34,721 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:34,721 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/B is initiating minor compaction (all files) 2024-11-16T14:52:34,721 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/A is initiating minor compaction (all files) 2024-11-16T14:52:34,721 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/A in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:34,721 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/B in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:34,721 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/cb4a363e5dae4f2eb8d69cd17618827f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/ca52d44b7cbf47b890b89895ffe3d544, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/0c09c504f1404beaacbebbd52cdf93a8] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=36.3 K 2024-11-16T14:52:34,721 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/83547d7d7bd649e19d51134005024da3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/959c9c87e575404a8c918d68a80f6bdd, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/aa65ba8ddae14eb5a9ded7c787d567b5] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=36.3 K 2024-11-16T14:52:34,722 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 83547d7d7bd649e19d51134005024da3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731768750830 2024-11-16T14:52:34,722 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb4a363e5dae4f2eb8d69cd17618827f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731768750830 2024-11-16T14:52:34,722 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 959c9c87e575404a8c918d68a80f6bdd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731768750901 2024-11-16T14:52:34,722 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca52d44b7cbf47b890b89895ffe3d544, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731768750901 2024-11-16T14:52:34,722 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting aa65ba8ddae14eb5a9ded7c787d567b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731768753110 2024-11-16T14:52:34,723 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c09c504f1404beaacbebbd52cdf93a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731768753110 2024-11-16T14:52:34,732 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:34,732 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/03b1b80510d84bd3904f1839b6977cea is 50, key is test_row_0/A:col10/1731768753432/Put/seqid=0 2024-11-16T14:52:34,741 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:34,741 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/2168125abd3d4ce1984a8eadf5112ef6 is 50, key is test_row_0/B:col10/1731768753432/Put/seqid=0 2024-11-16T14:52:34,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742103_1279 (size=12949) 2024-11-16T14:52:34,769 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/03b1b80510d84bd3904f1839b6977cea as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/03b1b80510d84bd3904f1839b6977cea 2024-11-16T14:52:34,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742104_1280 (size=12949) 2024-11-16T14:52:34,776 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/A of 0aa24bef827bbc1122cec8569b1d4491 into 03b1b80510d84bd3904f1839b6977cea(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:34,776 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:34,776 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/A, priority=13, startTime=1731768754720; duration=0sec 2024-11-16T14:52:34,776 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:34,776 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:A 2024-11-16T14:52:34,777 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:34,779 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:34,779 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/C is initiating minor compaction (all files) 2024-11-16T14:52:34,779 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/C in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:34,779 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7ae9b81771e34aee88fdd6175a1a7c3d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b2cfd753900f40cb849b43ed52a0d2d1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b6922e3ed468410896d55477f2dad08e] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=36.3 K 2024-11-16T14:52:34,780 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ae9b81771e34aee88fdd6175a1a7c3d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731768750830 2024-11-16T14:52:34,780 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2cfd753900f40cb849b43ed52a0d2d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731768750901 2024-11-16T14:52:34,783 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6922e3ed468410896d55477f2dad08e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731768753110 2024-11-16T14:52:34,785 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/2168125abd3d4ce1984a8eadf5112ef6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2168125abd3d4ce1984a8eadf5112ef6 2024-11-16T14:52:34,791 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/B of 0aa24bef827bbc1122cec8569b1d4491 into 2168125abd3d4ce1984a8eadf5112ef6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:34,791 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:34,791 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/B, priority=13, startTime=1731768754720; duration=0sec 2024-11-16T14:52:34,791 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:34,791 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:B 2024-11-16T14:52:34,808 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:34,809 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/c0f4793ef540474eb982311e1c4c3b3c is 50, key is test_row_0/C:col10/1731768753432/Put/seqid=0 2024-11-16T14:52:34,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742105_1281 (size=12949) 2024-11-16T14:52:34,854 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/c0f4793ef540474eb982311e1c4c3b3c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c0f4793ef540474eb982311e1c4c3b3c 2024-11-16T14:52:34,863 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/C of 0aa24bef827bbc1122cec8569b1d4491 into c0f4793ef540474eb982311e1c4c3b3c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:34,863 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:34,863 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/C, priority=13, startTime=1731768754720; duration=0sec 2024-11-16T14:52:34,863 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:34,863 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:C 2024-11-16T14:52:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:35,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:52:35,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:35,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:35,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:35,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:35,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:35,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:35,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/98d439a50f744c08b5588531c7416c17 is 50, key is test_row_0/A:col10/1731768755122/Put/seqid=0 2024-11-16T14:52:35,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742106_1282 (size=14741) 2024-11-16T14:52:35,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/98d439a50f744c08b5588531c7416c17 2024-11-16T14:52:35,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/3836ffd814324ad6a27b3d00d99bceb0 is 50, key is test_row_0/B:col10/1731768755122/Put/seqid=0 2024-11-16T14:52:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T14:52:35,172 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:35,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-16T14:52:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:52:35,174 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:35,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768815170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:35,175 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:35,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:35,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742107_1283 (size=12301) 2024-11-16T14:52:35,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/3836ffd814324ad6a27b3d00d99bceb0 2024-11-16T14:52:35,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/967be0089da142ec90457d1fa11ebf83 is 50, key is test_row_0/C:col10/1731768755122/Put/seqid=0 2024-11-16T14:52:35,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742108_1284 (size=12301) 2024-11-16T14:52:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:52:35,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768815281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:35,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T14:52:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:35,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:35,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:35,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:35,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:35,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T14:52:35,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:35,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:35,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:35,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:35,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:52:35,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768815491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:35,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768815620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:35,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768815620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:35,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768815620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:35,622 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in 0aa24bef827bbc1122cec8569b1d4491 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:35,623 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in 0aa24bef827bbc1122cec8569b1d4491 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:35,623 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in 0aa24bef827bbc1122cec8569b1d4491 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:35,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/967be0089da142ec90457d1fa11ebf83 2024-11-16T14:52:35,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/98d439a50f744c08b5588531c7416c17 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/98d439a50f744c08b5588531c7416c17 2024-11-16T14:52:35,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/98d439a50f744c08b5588531c7416c17, entries=200, sequenceid=290, filesize=14.4 K 2024-11-16T14:52:35,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T14:52:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:35,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:35,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/3836ffd814324ad6a27b3d00d99bceb0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/3836ffd814324ad6a27b3d00d99bceb0 2024-11-16T14:52:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:35,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/3836ffd814324ad6a27b3d00d99bceb0, entries=150, sequenceid=290, filesize=12.0 K 2024-11-16T14:52:35,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/967be0089da142ec90457d1fa11ebf83 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/967be0089da142ec90457d1fa11ebf83 2024-11-16T14:52:35,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/967be0089da142ec90457d1fa11ebf83, entries=150, sequenceid=290, filesize=12.0 K 2024-11-16T14:52:35,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0aa24bef827bbc1122cec8569b1d4491 in 537ms, sequenceid=290, compaction requested=false 2024-11-16T14:52:35,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:35,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T14:52:35,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:35,793 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:52:35,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:52:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:35,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:35,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/850e877c2c8143da98f74cb51a53e753 is 50, key is test_row_0/A:col10/1731768755168/Put/seqid=0 2024-11-16T14:52:35,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742109_1285 (size=12301) 2024-11-16T14:52:35,827 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/850e877c2c8143da98f74cb51a53e753 2024-11-16T14:52:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/62e9fae1b9d74d00b406352a9e56f777 is 50, key is test_row_0/B:col10/1731768755168/Put/seqid=0 2024-11-16T14:52:35,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742110_1286 (size=12301) 2024-11-16T14:52:35,860 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/62e9fae1b9d74d00b406352a9e56f777 2024-11-16T14:52:35,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/480bb16617284398ad2f2efd3ba66174 is 50, key is test_row_0/C:col10/1731768755168/Put/seqid=0 2024-11-16T14:52:35,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742111_1287 (size=12301) 2024-11-16T14:52:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768815907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:36,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768816020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:36,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:36,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:36,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:36,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768816231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:36,305 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/480bb16617284398ad2f2efd3ba66174 2024-11-16T14:52:36,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/850e877c2c8143da98f74cb51a53e753 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/850e877c2c8143da98f74cb51a53e753 2024-11-16T14:52:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:52:36,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/850e877c2c8143da98f74cb51a53e753, entries=150, sequenceid=302, filesize=12.0 K 2024-11-16T14:52:36,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/62e9fae1b9d74d00b406352a9e56f777 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/62e9fae1b9d74d00b406352a9e56f777 2024-11-16T14:52:36,318 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/62e9fae1b9d74d00b406352a9e56f777, entries=150, sequenceid=302, filesize=12.0 K 2024-11-16T14:52:36,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/480bb16617284398ad2f2efd3ba66174 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/480bb16617284398ad2f2efd3ba66174 2024-11-16T14:52:36,327 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/480bb16617284398ad2f2efd3ba66174, entries=150, sequenceid=302, filesize=12.0 K 2024-11-16T14:52:36,328 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0aa24bef827bbc1122cec8569b1d4491 in 535ms, sequenceid=302, compaction requested=true 2024-11-16T14:52:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-16T14:52:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-16T14:52:36,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-16T14:52:36,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1540 sec 2024-11-16T14:52:36,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.1590 sec 2024-11-16T14:52:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:36,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T14:52:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:36,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/caada8b82317451d97e49ad31c8383a4 is 50, key is test_row_0/A:col10/1731768755897/Put/seqid=0 2024-11-16T14:52:36,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742112_1288 (size=12301) 2024-11-16T14:52:36,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768816596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:36,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768816710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:36,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768816920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:36,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/caada8b82317451d97e49ad31c8383a4 2024-11-16T14:52:36,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/472bce669ec1418fa52fe91382c20f5e is 50, key is test_row_0/B:col10/1731768755897/Put/seqid=0 2024-11-16T14:52:37,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742113_1289 (size=12301) 2024-11-16T14:52:37,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/472bce669ec1418fa52fe91382c20f5e 2024-11-16T14:52:37,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/5b59852571ed4bf9b9f72a88826c11e9 is 50, key is test_row_0/C:col10/1731768755897/Put/seqid=0 2024-11-16T14:52:37,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742114_1290 (size=12301) 2024-11-16T14:52:37,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/5b59852571ed4bf9b9f72a88826c11e9 2024-11-16T14:52:37,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/caada8b82317451d97e49ad31c8383a4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/caada8b82317451d97e49ad31c8383a4 2024-11-16T14:52:37,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/caada8b82317451d97e49ad31c8383a4, entries=150, sequenceid=323, filesize=12.0 K 2024-11-16T14:52:37,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/472bce669ec1418fa52fe91382c20f5e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/472bce669ec1418fa52fe91382c20f5e 2024-11-16T14:52:37,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/472bce669ec1418fa52fe91382c20f5e, entries=150, sequenceid=323, filesize=12.0 K 2024-11-16T14:52:37,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/5b59852571ed4bf9b9f72a88826c11e9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5b59852571ed4bf9b9f72a88826c11e9 2024-11-16T14:52:37,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5b59852571ed4bf9b9f72a88826c11e9, entries=150, sequenceid=323, filesize=12.0 K 2024-11-16T14:52:37,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 0aa24bef827bbc1122cec8569b1d4491 in 521ms, sequenceid=323, compaction requested=true 2024-11-16T14:52:37,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:37,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:37,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:37,064 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:37,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:37,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:37,064 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:37,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:37,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:37,066 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:37,066 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/A is initiating minor compaction (all files) 2024-11-16T14:52:37,066 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/A in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:37,066 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/03b1b80510d84bd3904f1839b6977cea, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/98d439a50f744c08b5588531c7416c17, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/850e877c2c8143da98f74cb51a53e753, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/caada8b82317451d97e49ad31c8383a4] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=51.1 K 2024-11-16T14:52:37,067 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03b1b80510d84bd3904f1839b6977cea, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731768753110 2024-11-16T14:52:37,067 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:37,067 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/B is initiating minor compaction (all files) 2024-11-16T14:52:37,067 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/B in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:37,067 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2168125abd3d4ce1984a8eadf5112ef6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/3836ffd814324ad6a27b3d00d99bceb0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/62e9fae1b9d74d00b406352a9e56f777, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/472bce669ec1418fa52fe91382c20f5e] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=48.7 K 2024-11-16T14:52:37,068 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98d439a50f744c08b5588531c7416c17, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731768753452 2024-11-16T14:52:37,068 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2168125abd3d4ce1984a8eadf5112ef6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731768753110 2024-11-16T14:52:37,068 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 850e877c2c8143da98f74cb51a53e753, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731768755150 2024-11-16T14:52:37,068 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3836ffd814324ad6a27b3d00d99bceb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731768753452 2024-11-16T14:52:37,069 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting caada8b82317451d97e49ad31c8383a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731768755890 2024-11-16T14:52:37,069 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e9fae1b9d74d00b406352a9e56f777, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731768755150 2024-11-16T14:52:37,070 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 472bce669ec1418fa52fe91382c20f5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731768755890 2024-11-16T14:52:37,086 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:37,087 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/262d03376cdd4ff9a4daf102c0022138 is 50, key is test_row_0/B:col10/1731768755897/Put/seqid=0 2024-11-16T14:52:37,103 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:37,103 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/f5a1fddb655c4fc5af1d5f095be366de is 50, key is test_row_0/A:col10/1731768755897/Put/seqid=0 2024-11-16T14:52:37,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742115_1291 (size=13085) 2024-11-16T14:52:37,130 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/262d03376cdd4ff9a4daf102c0022138 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/262d03376cdd4ff9a4daf102c0022138 2024-11-16T14:52:37,138 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/B of 0aa24bef827bbc1122cec8569b1d4491 into 262d03376cdd4ff9a4daf102c0022138(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:37,138 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:37,138 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/B, priority=12, startTime=1731768757064; duration=0sec 2024-11-16T14:52:37,140 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:37,140 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:B 2024-11-16T14:52:37,141 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:37,143 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:37,143 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/C is initiating minor compaction (all files) 2024-11-16T14:52:37,143 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/C in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:37,143 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c0f4793ef540474eb982311e1c4c3b3c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/967be0089da142ec90457d1fa11ebf83, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/480bb16617284398ad2f2efd3ba66174, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5b59852571ed4bf9b9f72a88826c11e9] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=48.7 K 2024-11-16T14:52:37,144 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f4793ef540474eb982311e1c4c3b3c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731768753110 2024-11-16T14:52:37,144 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 967be0089da142ec90457d1fa11ebf83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731768753452 2024-11-16T14:52:37,145 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 480bb16617284398ad2f2efd3ba66174, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731768755150 2024-11-16T14:52:37,145 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b59852571ed4bf9b9f72a88826c11e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731768755890 2024-11-16T14:52:37,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742116_1292 (size=13085) 2024-11-16T14:52:37,158 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/f5a1fddb655c4fc5af1d5f095be366de as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f5a1fddb655c4fc5af1d5f095be366de 2024-11-16T14:52:37,164 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#C#compaction#251 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:37,164 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/A of 0aa24bef827bbc1122cec8569b1d4491 into f5a1fddb655c4fc5af1d5f095be366de(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:37,164 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:37,164 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/A, priority=12, startTime=1731768757064; duration=0sec 2024-11-16T14:52:37,165 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:37,165 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:A 2024-11-16T14:52:37,165 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/2be171c401aa4f379557e459e632916c is 50, key is test_row_0/C:col10/1731768755897/Put/seqid=0 2024-11-16T14:52:37,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742117_1293 (size=13085) 2024-11-16T14:52:37,195 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/2be171c401aa4f379557e459e632916c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/2be171c401aa4f379557e459e632916c 2024-11-16T14:52:37,205 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/C of 0aa24bef827bbc1122cec8569b1d4491 into 2be171c401aa4f379557e459e632916c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:37,205 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:37,205 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/C, priority=12, startTime=1731768757064; duration=0sec 2024-11-16T14:52:37,205 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:37,205 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:C 2024-11-16T14:52:37,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:52:37,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:37,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:37,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:37,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:37,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:37,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:37,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/5ff8ae8c4ebd44ddb5a67379ef9230e1 is 50, key is test_row_0/A:col10/1731768756594/Put/seqid=0 2024-11-16T14:52:37,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742118_1294 (size=12297) 2024-11-16T14:52:37,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/5ff8ae8c4ebd44ddb5a67379ef9230e1 2024-11-16T14:52:37,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/8d60737b34854544967dc6619b5c40f6 is 50, key is test_row_0/B:col10/1731768756594/Put/seqid=0 2024-11-16T14:52:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T14:52:37,321 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:37,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-16T14:52:37,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742119_1295 (size=9857) 2024-11-16T14:52:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:52:37,323 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:37,324 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:37,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:37,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/8d60737b34854544967dc6619b5c40f6 2024-11-16T14:52:37,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768817329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:37,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/a1e854ca363e434bb8e8786588dc0fec is 50, key is test_row_0/C:col10/1731768756594/Put/seqid=0 2024-11-16T14:52:37,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742120_1296 (size=9857) 2024-11-16T14:52:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:52:37,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768817452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:37,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T14:52:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:37,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:37,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60216 deadline: 1731768817631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:37,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T14:52:37,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:37,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:37,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:37,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:37,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:37,632 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in 0aa24bef827bbc1122cec8569b1d4491 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:52:37,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60224 deadline: 1731768817640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:37,643 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in 0aa24bef827bbc1122cec8569b1d4491 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:37,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60186 deadline: 1731768817651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:37,653 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in 0aa24bef827bbc1122cec8569b1d4491 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:37,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:37,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768817661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:37,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T14:52:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:37,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:37,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/a1e854ca363e434bb8e8786588dc0fec 2024-11-16T14:52:37,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/5ff8ae8c4ebd44ddb5a67379ef9230e1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/5ff8ae8c4ebd44ddb5a67379ef9230e1 2024-11-16T14:52:37,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/5ff8ae8c4ebd44ddb5a67379ef9230e1, entries=150, sequenceid=338, filesize=12.0 K 2024-11-16T14:52:37,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/8d60737b34854544967dc6619b5c40f6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8d60737b34854544967dc6619b5c40f6 2024-11-16T14:52:37,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8d60737b34854544967dc6619b5c40f6, entries=100, sequenceid=338, filesize=9.6 K 2024-11-16T14:52:37,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/a1e854ca363e434bb8e8786588dc0fec as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/a1e854ca363e434bb8e8786588dc0fec 2024-11-16T14:52:37,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/a1e854ca363e434bb8e8786588dc0fec, entries=100, sequenceid=338, filesize=9.6 K 2024-11-16T14:52:37,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0aa24bef827bbc1122cec8569b1d4491 in 581ms, sequenceid=338, compaction requested=false 2024-11-16T14:52:37,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:37,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T14:52:37,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:37,938 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:52:37,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:37,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:37,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:37,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:37,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:37,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:37,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/29d22f53c6ad4f3bbd225e1f784e0abf is 50, key is test_row_0/A:col10/1731768757327/Put/seqid=0 2024-11-16T14:52:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:52:37,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742121_1297 (size=12301) 2024-11-16T14:52:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:37,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:38,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768818011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:38,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768818120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:38,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768818330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:38,354 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/29d22f53c6ad4f3bbd225e1f784e0abf 2024-11-16T14:52:38,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/7222f0bc3b1145b2ab5e7b55e2910669 is 50, key is test_row_0/B:col10/1731768757327/Put/seqid=0 2024-11-16T14:52:38,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742122_1298 (size=12301) 2024-11-16T14:52:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:52:38,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768818641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:38,773 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/7222f0bc3b1145b2ab5e7b55e2910669 2024-11-16T14:52:38,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/af7beeb455c14e9b93ebcdff25b6730b is 50, key is test_row_0/C:col10/1731768757327/Put/seqid=0 2024-11-16T14:52:38,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742123_1299 (size=12301) 2024-11-16T14:52:38,809 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/af7beeb455c14e9b93ebcdff25b6730b 2024-11-16T14:52:38,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/29d22f53c6ad4f3bbd225e1f784e0abf as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/29d22f53c6ad4f3bbd225e1f784e0abf 2024-11-16T14:52:38,823 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/29d22f53c6ad4f3bbd225e1f784e0abf, entries=150, sequenceid=358, filesize=12.0 K 2024-11-16T14:52:38,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/7222f0bc3b1145b2ab5e7b55e2910669 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/7222f0bc3b1145b2ab5e7b55e2910669 2024-11-16T14:52:38,830 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/7222f0bc3b1145b2ab5e7b55e2910669, entries=150, sequenceid=358, filesize=12.0 K 2024-11-16T14:52:38,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/af7beeb455c14e9b93ebcdff25b6730b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/af7beeb455c14e9b93ebcdff25b6730b 2024-11-16T14:52:38,844 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/af7beeb455c14e9b93ebcdff25b6730b, entries=150, sequenceid=358, filesize=12.0 K 2024-11-16T14:52:38,845 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0aa24bef827bbc1122cec8569b1d4491 in 907ms, sequenceid=358, compaction requested=true 2024-11-16T14:52:38,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:38,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:38,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-16T14:52:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-16T14:52:38,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-16T14:52:38,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5230 sec 2024-11-16T14:52:38,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5290 sec 2024-11-16T14:52:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:39,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:52:39,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:39,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:39,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:39,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:39,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:39,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:39,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/6fe75847fc1d4e368f3442acc4395b15 is 50, key is test_row_0/A:col10/1731768758005/Put/seqid=0 2024-11-16T14:52:39,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742124_1300 (size=14741) 2024-11-16T14:52:39,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/6fe75847fc1d4e368f3442acc4395b15 2024-11-16T14:52:39,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/030b7da1d2774c41b458bfc87a8f37f1 is 50, key is test_row_0/B:col10/1731768758005/Put/seqid=0 2024-11-16T14:52:39,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742125_1301 (size=12301) 2024-11-16T14:52:39,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/030b7da1d2774c41b458bfc87a8f37f1 2024-11-16T14:52:39,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/42d40bf41697425d8ea6b54f287b1088 is 50, key is test_row_0/C:col10/1731768758005/Put/seqid=0 2024-11-16T14:52:39,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768819280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:39,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:39,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:39,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:39,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742126_1302 (size=12301) 2024-11-16T14:52:39,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/42d40bf41697425d8ea6b54f287b1088 2024-11-16T14:52:39,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/6fe75847fc1d4e368f3442acc4395b15 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6fe75847fc1d4e368f3442acc4395b15 2024-11-16T14:52:39,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6fe75847fc1d4e368f3442acc4395b15, entries=200, sequenceid=371, filesize=14.4 K 2024-11-16T14:52:39,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/030b7da1d2774c41b458bfc87a8f37f1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/030b7da1d2774c41b458bfc87a8f37f1 2024-11-16T14:52:39,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/030b7da1d2774c41b458bfc87a8f37f1, entries=150, sequenceid=371, filesize=12.0 K 2024-11-16T14:52:39,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/42d40bf41697425d8ea6b54f287b1088 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/42d40bf41697425d8ea6b54f287b1088 2024-11-16T14:52:39,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/42d40bf41697425d8ea6b54f287b1088, entries=150, sequenceid=371, filesize=12.0 K 2024-11-16T14:52:39,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0aa24bef827bbc1122cec8569b1d4491 in 167ms, sequenceid=371, compaction requested=true 2024-11-16T14:52:39,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:39,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:39,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:39,331 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:39,331 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:39,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:39,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:39,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0aa24bef827bbc1122cec8569b1d4491:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:39,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:39,335 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:39,335 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/B is initiating minor compaction (all files) 2024-11-16T14:52:39,335 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/B in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:39,335 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/262d03376cdd4ff9a4daf102c0022138, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8d60737b34854544967dc6619b5c40f6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/7222f0bc3b1145b2ab5e7b55e2910669, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/030b7da1d2774c41b458bfc87a8f37f1] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=46.4 K 2024-11-16T14:52:39,335 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52424 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:39,335 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/A is initiating minor compaction (all files) 2024-11-16T14:52:39,335 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/A in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:39,335 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f5a1fddb655c4fc5af1d5f095be366de, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/5ff8ae8c4ebd44ddb5a67379ef9230e1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/29d22f53c6ad4f3bbd225e1f784e0abf, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6fe75847fc1d4e368f3442acc4395b15] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=51.2 K 2024-11-16T14:52:39,336 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a1fddb655c4fc5af1d5f095be366de, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731768755890 2024-11-16T14:52:39,336 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 262d03376cdd4ff9a4daf102c0022138, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731768755890 2024-11-16T14:52:39,336 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ff8ae8c4ebd44ddb5a67379ef9230e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731768756581 2024-11-16T14:52:39,337 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d60737b34854544967dc6619b5c40f6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731768756594 2024-11-16T14:52:39,337 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d22f53c6ad4f3bbd225e1f784e0abf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731768757298 2024-11-16T14:52:39,337 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7222f0bc3b1145b2ab5e7b55e2910669, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731768757298 2024-11-16T14:52:39,337 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fe75847fc1d4e368f3442acc4395b15, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731768758000 2024-11-16T14:52:39,338 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 030b7da1d2774c41b458bfc87a8f37f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731768758000 2024-11-16T14:52:39,365 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:39,366 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/fa4ef75865194ae794b435ee6bde58b7 is 50, key is test_row_0/B:col10/1731768758005/Put/seqid=0 2024-11-16T14:52:39,373 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:39,374 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/4109d421fa774276ab8d4da19b0f362f is 50, key is test_row_0/A:col10/1731768758005/Put/seqid=0 2024-11-16T14:52:39,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742127_1303 (size=13221) 2024-11-16T14:52:39,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742128_1304 (size=13221) 2024-11-16T14:52:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:39,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:52:39,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:39,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:39,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:39,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:39,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:39,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:39,398 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/4109d421fa774276ab8d4da19b0f362f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/4109d421fa774276ab8d4da19b0f362f 2024-11-16T14:52:39,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/79d8c7bfcf7e4cabac9131e60267b818 is 50, key is test_row_0/A:col10/1731768759392/Put/seqid=0 2024-11-16T14:52:39,405 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/A of 0aa24bef827bbc1122cec8569b1d4491 into 4109d421fa774276ab8d4da19b0f362f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:39,405 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:39,405 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/A, priority=12, startTime=1731768759331; duration=0sec 2024-11-16T14:52:39,405 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:39,405 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:A 2024-11-16T14:52:39,405 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:39,406 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:39,406 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 0aa24bef827bbc1122cec8569b1d4491/C is initiating minor compaction (all files) 2024-11-16T14:52:39,406 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0aa24bef827bbc1122cec8569b1d4491/C in TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:39,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742129_1305 (size=14741) 2024-11-16T14:52:39,407 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/2be171c401aa4f379557e459e632916c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/a1e854ca363e434bb8e8786588dc0fec, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/af7beeb455c14e9b93ebcdff25b6730b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/42d40bf41697425d8ea6b54f287b1088] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp, totalSize=46.4 K 2024-11-16T14:52:39,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/79d8c7bfcf7e4cabac9131e60267b818 2024-11-16T14:52:39,408 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2be171c401aa4f379557e459e632916c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731768755890 2024-11-16T14:52:39,409 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1e854ca363e434bb8e8786588dc0fec, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731768756594 2024-11-16T14:52:39,409 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting af7beeb455c14e9b93ebcdff25b6730b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731768757298 2024-11-16T14:52:39,409 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d40bf41697425d8ea6b54f287b1088, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731768758000 2024-11-16T14:52:39,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/76aabc3447a94f9f9633b7414b433440 is 50, key is test_row_0/B:col10/1731768759392/Put/seqid=0 2024-11-16T14:52:39,418 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0aa24bef827bbc1122cec8569b1d4491#C#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:39,419 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/f2d2442a461e4e0fb961677865010950 is 50, key is test_row_0/C:col10/1731768758005/Put/seqid=0 2024-11-16T14:52:39,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742130_1306 (size=12301) 2024-11-16T14:52:39,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742131_1307 (size=13221) 2024-11-16T14:52:39,436 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/f2d2442a461e4e0fb961677865010950 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f2d2442a461e4e0fb961677865010950 2024-11-16T14:52:39,444 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/C of 0aa24bef827bbc1122cec8569b1d4491 into f2d2442a461e4e0fb961677865010950(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:39,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:39,444 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/C, priority=12, startTime=1731768759332; duration=0sec 2024-11-16T14:52:39,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:39,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:C 2024-11-16T14:52:39,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768819453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T14:52:39,481 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:39,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-16T14:52:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:52:39,483 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:39,484 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:39,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:39,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768819561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:52:39,625 INFO [Thread-1062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1062. 2024-11-16T14:52:39,625 DEBUG [Thread-1062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:39,625 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:39,625 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:39,626 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:39,628 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-16T14:52:39,628 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-16T14:52:39,628 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:39,628 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:39,628 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:39,628 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:39,629 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:39,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:39,630 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-16T14:52:39,630 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:39,630 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:39,630 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:39,630 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:39,631 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:39,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:39,633 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-16T14:52:39,633 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:39,633 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:39,633 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:39,633 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:39,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:52:39,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:39,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:39,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:39,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:39,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:39,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768819770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:39,789 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/fa4ef75865194ae794b435ee6bde58b7 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/fa4ef75865194ae794b435ee6bde58b7 2024-11-16T14:52:39,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:52:39,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:39,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:39,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:39,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:39,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:39,794 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0aa24bef827bbc1122cec8569b1d4491/B of 0aa24bef827bbc1122cec8569b1d4491 into fa4ef75865194ae794b435ee6bde58b7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:39,794 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:39,794 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., storeName=0aa24bef827bbc1122cec8569b1d4491/B, priority=12, startTime=1731768759331; duration=0sec 2024-11-16T14:52:39,794 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:39,794 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0aa24bef827bbc1122cec8569b1d4491:B 2024-11-16T14:52:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:52:39,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/76aabc3447a94f9f9633b7414b433440 2024-11-16T14:52:39,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/9dc07c447c7a406d91ffdcf50f3ea42e is 50, key is test_row_0/C:col10/1731768759392/Put/seqid=0 2024-11-16T14:52:39,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742132_1308 (size=12301) 2024-11-16T14:52:39,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:52:39,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:39,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:39,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:39,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:39,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:40,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60184 deadline: 1731768820081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:52:40,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:52:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:40,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:52:40,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/9dc07c447c7a406d91ffdcf50f3ea42e 2024-11-16T14:52:40,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/79d8c7bfcf7e4cabac9131e60267b818 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/79d8c7bfcf7e4cabac9131e60267b818 2024-11-16T14:52:40,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/79d8c7bfcf7e4cabac9131e60267b818, entries=200, sequenceid=391, filesize=14.4 K 2024-11-16T14:52:40,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/76aabc3447a94f9f9633b7414b433440 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/76aabc3447a94f9f9633b7414b433440 2024-11-16T14:52:40,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:52:40,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:40,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:40,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:40,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:40,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:40,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/76aabc3447a94f9f9633b7414b433440, entries=150, sequenceid=391, filesize=12.0 K 2024-11-16T14:52:40,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/9dc07c447c7a406d91ffdcf50f3ea42e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/9dc07c447c7a406d91ffdcf50f3ea42e 2024-11-16T14:52:40,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/9dc07c447c7a406d91ffdcf50f3ea42e, entries=150, sequenceid=391, filesize=12.0 K 2024-11-16T14:52:40,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0aa24bef827bbc1122cec8569b1d4491 in 867ms, sequenceid=391, compaction requested=false 2024-11-16T14:52:40,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:40,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T14:52:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:40,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:52:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:40,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:40,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:40,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/c0040dd042a446e98c7d66f465fb1f09 is 50, key is test_row_0/A:col10/1731768759411/Put/seqid=0 2024-11-16T14:52:40,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742133_1309 (size=12301) 2024-11-16T14:52:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:40,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. as already flushing 2024-11-16T14:52:40,592 INFO [Thread-1049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1049. 2024-11-16T14:52:40,593 DEBUG [Thread-1049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:40,593 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:40,593 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:40,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:52:40,825 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/c0040dd042a446e98c7d66f465fb1f09 2024-11-16T14:52:40,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/58a04e5077494901ba31ab35a9db5840 is 50, key is test_row_0/B:col10/1731768759411/Put/seqid=0 2024-11-16T14:52:40,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742134_1310 (size=12301) 2024-11-16T14:52:41,235 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/58a04e5077494901ba31ab35a9db5840 2024-11-16T14:52:41,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/7fbcb29d38ad495cbe08762ed7bef861 is 50, key is test_row_0/C:col10/1731768759411/Put/seqid=0 2024-11-16T14:52:41,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742135_1311 (size=12301) 2024-11-16T14:52:41,592 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-11-16T14:52:41,592 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:41,592 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:41,592 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:41,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:52:41,642 INFO [Thread-1047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1047. 2024-11-16T14:52:41,643 DEBUG [Thread-1047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:41,643 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:41,643 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:41,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:41,648 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/7fbcb29d38ad495cbe08762ed7bef861 2024-11-16T14:52:41,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/c0040dd042a446e98c7d66f465fb1f09 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/c0040dd042a446e98c7d66f465fb1f09 2024-11-16T14:52:41,659 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/c0040dd042a446e98c7d66f465fb1f09, entries=150, sequenceid=406, filesize=12.0 K 2024-11-16T14:52:41,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/58a04e5077494901ba31ab35a9db5840 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/58a04e5077494901ba31ab35a9db5840 2024-11-16T14:52:41,663 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/58a04e5077494901ba31ab35a9db5840, entries=150, sequenceid=406, filesize=12.0 K 2024-11-16T14:52:41,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/7fbcb29d38ad495cbe08762ed7bef861 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7fbcb29d38ad495cbe08762ed7bef861 2024-11-16T14:52:41,668 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7fbcb29d38ad495cbe08762ed7bef861, entries=150, sequenceid=406, filesize=12.0 K 2024-11-16T14:52:41,669 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 0aa24bef827bbc1122cec8569b1d4491 in 1264ms, sequenceid=406, compaction requested=true 2024-11-16T14:52:41,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 0aa24bef827bbc1122cec8569b1d4491: 2024-11-16T14:52:41,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:41,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-16T14:52:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-16T14:52:41,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-16T14:52:41,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1860 sec 2024-11-16T14:52:41,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.1930 sec 2024-11-16T14:52:41,692 INFO [Thread-1045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1045. 2024-11-16T14:52:41,692 INFO [Thread-1051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1051. 2024-11-16T14:52:41,692 DEBUG [Thread-1051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:41,692 DEBUG [Thread-1045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:52:41,693 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:41,693 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:41,693 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:41,693 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:41,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:41,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:41,710 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:52:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T14:52:43,641 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2702 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2731 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2827 2024-11-16T14:52:43,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2729 2024-11-16T14:52:43,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2739 2024-11-16T14:52:43,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:52:43,642 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:52:43,642 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:52:43,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:43,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:43,642 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:52:43,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:52:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:52:43,648 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768763648"}]},"ts":"1731768763648"} 2024-11-16T14:52:43,651 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:52:43,651 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:52:43,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:52:43,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0aa24bef827bbc1122cec8569b1d4491, UNASSIGN}] 2024-11-16T14:52:43,655 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0aa24bef827bbc1122cec8569b1d4491, UNASSIGN 2024-11-16T14:52:43,656 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=0aa24bef827bbc1122cec8569b1d4491, regionState=CLOSING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:43,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0aa24bef827bbc1122cec8569b1d4491, UNASSIGN because future has completed 2024-11-16T14:52:43,658 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:52:43,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:52:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:52:43,811 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:43,811 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:52:43,811 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 0aa24bef827bbc1122cec8569b1d4491, disabling compactions & flushes 2024-11-16T14:52:43,811 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:43,811 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:43,811 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. after waiting 0 ms 2024-11-16T14:52:43,811 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:43,811 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 0aa24bef827bbc1122cec8569b1d4491 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T14:52:43,812 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=A 2024-11-16T14:52:43,812 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:43,812 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=B 2024-11-16T14:52:43,812 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:43,812 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0aa24bef827bbc1122cec8569b1d4491, store=C 2024-11-16T14:52:43,812 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:43,819 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/8add71eb9d76457f93348188b9e2e5c0 is 50, key is test_row_1/A:col10/1731768761691/Put/seqid=0 2024-11-16T14:52:43,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742136_1312 (size=9857) 2024-11-16T14:52:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:52:44,223 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/8add71eb9d76457f93348188b9e2e5c0 2024-11-16T14:52:44,230 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/f12793f6af6a415a8d767d589af38ad9 is 50, key is test_row_1/B:col10/1731768761691/Put/seqid=0 2024-11-16T14:52:44,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742137_1313 (size=9857) 2024-11-16T14:52:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:52:44,634 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/f12793f6af6a415a8d767d589af38ad9 2024-11-16T14:52:44,645 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/d6db674b45384ca2903dbf9a41dcbbaf is 50, key is test_row_1/C:col10/1731768761691/Put/seqid=0 2024-11-16T14:52:44,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742138_1314 (size=9857) 2024-11-16T14:52:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:52:45,049 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/d6db674b45384ca2903dbf9a41dcbbaf 2024-11-16T14:52:45,053 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/A/8add71eb9d76457f93348188b9e2e5c0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/8add71eb9d76457f93348188b9e2e5c0 2024-11-16T14:52:45,056 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/8add71eb9d76457f93348188b9e2e5c0, entries=100, sequenceid=414, filesize=9.6 K 2024-11-16T14:52:45,057 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/B/f12793f6af6a415a8d767d589af38ad9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/f12793f6af6a415a8d767d589af38ad9 2024-11-16T14:52:45,060 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/f12793f6af6a415a8d767d589af38ad9, entries=100, sequenceid=414, filesize=9.6 K 2024-11-16T14:52:45,061 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/.tmp/C/d6db674b45384ca2903dbf9a41dcbbaf as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/d6db674b45384ca2903dbf9a41dcbbaf 2024-11-16T14:52:45,064 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/d6db674b45384ca2903dbf9a41dcbbaf, entries=100, sequenceid=414, filesize=9.6 K 2024-11-16T14:52:45,065 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0aa24bef827bbc1122cec8569b1d4491 in 1254ms, sequenceid=414, compaction requested=true 2024-11-16T14:52:45,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/34b47461cbd64364a095c46a9bf28aaf, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/025dc0e818bc4b26abd3b7b3002234d9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/866e7a438824431691c103ded9de748a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/52b92e90c784433eb478fc766cd1bdf2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1a4b6ed25c2943ca86d4477aedcf73e1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f882ccf9ac23459c94f1819fc9a34f09, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/b68038045eb04676af56b6070e56b914, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/9efd082f5e124d088f1cd362075eba3d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6620e262fed247ba96582f50bd76b5e0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1d0c26c54e7546bdb4f4587f02e677a1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/20260297f1974162812f7e1b47af03eb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/87dc311d1bd64edebdd80517dd1fac25, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/860d5636d0974a15898c9fb844114a12, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/c693a314614b45bea8436c693a225181, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/42847b07edb74148a80d77fccbed533a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/cb4a363e5dae4f2eb8d69cd17618827f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/258b62901ecd4016ae7652660f25f806, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/ca52d44b7cbf47b890b89895ffe3d544, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/03b1b80510d84bd3904f1839b6977cea, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/0c09c504f1404beaacbebbd52cdf93a8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/98d439a50f744c08b5588531c7416c17, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/850e877c2c8143da98f74cb51a53e753, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f5a1fddb655c4fc5af1d5f095be366de, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/caada8b82317451d97e49ad31c8383a4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/5ff8ae8c4ebd44ddb5a67379ef9230e1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/29d22f53c6ad4f3bbd225e1f784e0abf, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6fe75847fc1d4e368f3442acc4395b15] to archive 2024-11-16T14:52:45,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:52:45,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/34b47461cbd64364a095c46a9bf28aaf to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/34b47461cbd64364a095c46a9bf28aaf 2024-11-16T14:52:45,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/025dc0e818bc4b26abd3b7b3002234d9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/025dc0e818bc4b26abd3b7b3002234d9 2024-11-16T14:52:45,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/866e7a438824431691c103ded9de748a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/866e7a438824431691c103ded9de748a 2024-11-16T14:52:45,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/52b92e90c784433eb478fc766cd1bdf2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/52b92e90c784433eb478fc766cd1bdf2 2024-11-16T14:52:45,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1a4b6ed25c2943ca86d4477aedcf73e1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1a4b6ed25c2943ca86d4477aedcf73e1 2024-11-16T14:52:45,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f882ccf9ac23459c94f1819fc9a34f09 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f882ccf9ac23459c94f1819fc9a34f09 2024-11-16T14:52:45,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/b68038045eb04676af56b6070e56b914 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/b68038045eb04676af56b6070e56b914 2024-11-16T14:52:45,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/9efd082f5e124d088f1cd362075eba3d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/9efd082f5e124d088f1cd362075eba3d 2024-11-16T14:52:45,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6620e262fed247ba96582f50bd76b5e0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6620e262fed247ba96582f50bd76b5e0 2024-11-16T14:52:45,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1d0c26c54e7546bdb4f4587f02e677a1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/1d0c26c54e7546bdb4f4587f02e677a1 2024-11-16T14:52:45,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/20260297f1974162812f7e1b47af03eb to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/20260297f1974162812f7e1b47af03eb 2024-11-16T14:52:45,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/87dc311d1bd64edebdd80517dd1fac25 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/87dc311d1bd64edebdd80517dd1fac25 2024-11-16T14:52:45,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/860d5636d0974a15898c9fb844114a12 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/860d5636d0974a15898c9fb844114a12 2024-11-16T14:52:45,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/c693a314614b45bea8436c693a225181 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/c693a314614b45bea8436c693a225181 2024-11-16T14:52:45,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/42847b07edb74148a80d77fccbed533a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/42847b07edb74148a80d77fccbed533a 2024-11-16T14:52:45,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/cb4a363e5dae4f2eb8d69cd17618827f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/cb4a363e5dae4f2eb8d69cd17618827f 2024-11-16T14:52:45,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/258b62901ecd4016ae7652660f25f806 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/258b62901ecd4016ae7652660f25f806 2024-11-16T14:52:45,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/ca52d44b7cbf47b890b89895ffe3d544 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/ca52d44b7cbf47b890b89895ffe3d544 2024-11-16T14:52:45,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/03b1b80510d84bd3904f1839b6977cea to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/03b1b80510d84bd3904f1839b6977cea 2024-11-16T14:52:45,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/0c09c504f1404beaacbebbd52cdf93a8 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/0c09c504f1404beaacbebbd52cdf93a8 2024-11-16T14:52:45,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/98d439a50f744c08b5588531c7416c17 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/98d439a50f744c08b5588531c7416c17 2024-11-16T14:52:45,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/850e877c2c8143da98f74cb51a53e753 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/850e877c2c8143da98f74cb51a53e753 2024-11-16T14:52:45,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f5a1fddb655c4fc5af1d5f095be366de to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/f5a1fddb655c4fc5af1d5f095be366de 2024-11-16T14:52:45,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/caada8b82317451d97e49ad31c8383a4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/caada8b82317451d97e49ad31c8383a4 2024-11-16T14:52:45,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/5ff8ae8c4ebd44ddb5a67379ef9230e1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/5ff8ae8c4ebd44ddb5a67379ef9230e1 2024-11-16T14:52:45,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/29d22f53c6ad4f3bbd225e1f784e0abf to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/29d22f53c6ad4f3bbd225e1f784e0abf 2024-11-16T14:52:45,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6fe75847fc1d4e368f3442acc4395b15 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/6fe75847fc1d4e368f3442acc4395b15 2024-11-16T14:52:45,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8a3eb24c4ac34216a025057536f6941b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/86ad886284a64ac0b204f306879d6228, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/663c87aa97b0478bbbf7cf63b33c4d24, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/9432b89b059e4bbda7fc78e2b5475281, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/57e21ee468df4fefbc8c850c0e0a4254, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/88c8033d40a1485985e342ea30be281b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2ad3b8eeeb8046e1b0a322e603207b3c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/78401962a24d4722a9ebcb7d74c92cea, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2aa3e8f757c94ce18692e3143aa1f7f5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/658c8cdf3a9546f5ae1a00f467fbe6c0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e8eae55971504f828a8431de04240bee, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/11863d754dfe4b46a787d4308f4335f2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/63e78c94c2234f8283fcfb3fb89384f3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/afe81161d6de4d94a35de49fbf8a3195, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e848b25c68aa4060910c48e950452eac, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/83547d7d7bd649e19d51134005024da3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/09f02e2aaeff42acaf4d2c92c9a03ede, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/959c9c87e575404a8c918d68a80f6bdd, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2168125abd3d4ce1984a8eadf5112ef6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/aa65ba8ddae14eb5a9ded7c787d567b5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/3836ffd814324ad6a27b3d00d99bceb0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/62e9fae1b9d74d00b406352a9e56f777, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/262d03376cdd4ff9a4daf102c0022138, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/472bce669ec1418fa52fe91382c20f5e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8d60737b34854544967dc6619b5c40f6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/7222f0bc3b1145b2ab5e7b55e2910669, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/030b7da1d2774c41b458bfc87a8f37f1] to archive 2024-11-16T14:52:45,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:52:45,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8a3eb24c4ac34216a025057536f6941b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8a3eb24c4ac34216a025057536f6941b 2024-11-16T14:52:45,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/86ad886284a64ac0b204f306879d6228 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/86ad886284a64ac0b204f306879d6228 2024-11-16T14:52:45,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/663c87aa97b0478bbbf7cf63b33c4d24 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/663c87aa97b0478bbbf7cf63b33c4d24 2024-11-16T14:52:45,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/9432b89b059e4bbda7fc78e2b5475281 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/9432b89b059e4bbda7fc78e2b5475281 2024-11-16T14:52:45,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/57e21ee468df4fefbc8c850c0e0a4254 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/57e21ee468df4fefbc8c850c0e0a4254 2024-11-16T14:52:45,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/88c8033d40a1485985e342ea30be281b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/88c8033d40a1485985e342ea30be281b 2024-11-16T14:52:45,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2ad3b8eeeb8046e1b0a322e603207b3c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2ad3b8eeeb8046e1b0a322e603207b3c 2024-11-16T14:52:45,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/78401962a24d4722a9ebcb7d74c92cea to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/78401962a24d4722a9ebcb7d74c92cea 2024-11-16T14:52:45,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2aa3e8f757c94ce18692e3143aa1f7f5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2aa3e8f757c94ce18692e3143aa1f7f5 2024-11-16T14:52:45,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/658c8cdf3a9546f5ae1a00f467fbe6c0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/658c8cdf3a9546f5ae1a00f467fbe6c0 2024-11-16T14:52:45,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e8eae55971504f828a8431de04240bee to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e8eae55971504f828a8431de04240bee 2024-11-16T14:52:45,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/11863d754dfe4b46a787d4308f4335f2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/11863d754dfe4b46a787d4308f4335f2 2024-11-16T14:52:45,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/63e78c94c2234f8283fcfb3fb89384f3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/63e78c94c2234f8283fcfb3fb89384f3 2024-11-16T14:52:45,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/afe81161d6de4d94a35de49fbf8a3195 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/afe81161d6de4d94a35de49fbf8a3195 2024-11-16T14:52:45,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e848b25c68aa4060910c48e950452eac to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/e848b25c68aa4060910c48e950452eac 2024-11-16T14:52:45,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/83547d7d7bd649e19d51134005024da3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/83547d7d7bd649e19d51134005024da3 2024-11-16T14:52:45,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/09f02e2aaeff42acaf4d2c92c9a03ede to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/09f02e2aaeff42acaf4d2c92c9a03ede 2024-11-16T14:52:45,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/959c9c87e575404a8c918d68a80f6bdd to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/959c9c87e575404a8c918d68a80f6bdd 2024-11-16T14:52:45,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2168125abd3d4ce1984a8eadf5112ef6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/2168125abd3d4ce1984a8eadf5112ef6 2024-11-16T14:52:45,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/aa65ba8ddae14eb5a9ded7c787d567b5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/aa65ba8ddae14eb5a9ded7c787d567b5 2024-11-16T14:52:45,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/3836ffd814324ad6a27b3d00d99bceb0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/3836ffd814324ad6a27b3d00d99bceb0 2024-11-16T14:52:45,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/62e9fae1b9d74d00b406352a9e56f777 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/62e9fae1b9d74d00b406352a9e56f777 2024-11-16T14:52:45,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/262d03376cdd4ff9a4daf102c0022138 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/262d03376cdd4ff9a4daf102c0022138 2024-11-16T14:52:45,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/472bce669ec1418fa52fe91382c20f5e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/472bce669ec1418fa52fe91382c20f5e 2024-11-16T14:52:45,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8d60737b34854544967dc6619b5c40f6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/8d60737b34854544967dc6619b5c40f6 2024-11-16T14:52:45,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/7222f0bc3b1145b2ab5e7b55e2910669 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/7222f0bc3b1145b2ab5e7b55e2910669 2024-11-16T14:52:45,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/030b7da1d2774c41b458bfc87a8f37f1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/030b7da1d2774c41b458bfc87a8f37f1 2024-11-16T14:52:45,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/61fa3fc05a404b24a3ff76b9cb51370d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/0402ef749d6c493fb55804afdd66cb19, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/30aa8450bad24658a95360cf65f93079, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f92e788d3d6f4da4a2245e2e9807f605, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f5d7e8f575ab43adba7a2c815cd3d70f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/29c12a137e624439a9f8f519c8afda78, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e2311db667634a3f8db9f466b4a04515, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c602113ffa90444daedb46e04a2983d6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/400757b47500421e87295e9d1abbd89e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/4f57d0b05f0d4e38b75e6846e8ceddb0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1c638c6f73444d3eb701c9a4fe44fd14, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e7953bb07304481d903d8717a9f1a9a9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7e72509d5da1499f80c944655e927551, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5557f5bd5d504fc491c1be20fd1658e9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1a19db89278b497184636b526bb7e0cb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7ae9b81771e34aee88fdd6175a1a7c3d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b9ebe63117a64969b1757d9d0848dc94, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b2cfd753900f40cb849b43ed52a0d2d1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c0f4793ef540474eb982311e1c4c3b3c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b6922e3ed468410896d55477f2dad08e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/967be0089da142ec90457d1fa11ebf83, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/480bb16617284398ad2f2efd3ba66174, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/2be171c401aa4f379557e459e632916c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5b59852571ed4bf9b9f72a88826c11e9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/a1e854ca363e434bb8e8786588dc0fec, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/af7beeb455c14e9b93ebcdff25b6730b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/42d40bf41697425d8ea6b54f287b1088] to archive 2024-11-16T14:52:45,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:52:45,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/61fa3fc05a404b24a3ff76b9cb51370d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/61fa3fc05a404b24a3ff76b9cb51370d 2024-11-16T14:52:45,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/0402ef749d6c493fb55804afdd66cb19 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/0402ef749d6c493fb55804afdd66cb19 2024-11-16T14:52:45,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/30aa8450bad24658a95360cf65f93079 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/30aa8450bad24658a95360cf65f93079 2024-11-16T14:52:45,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f92e788d3d6f4da4a2245e2e9807f605 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f92e788d3d6f4da4a2245e2e9807f605 2024-11-16T14:52:45,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f5d7e8f575ab43adba7a2c815cd3d70f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f5d7e8f575ab43adba7a2c815cd3d70f 2024-11-16T14:52:45,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/29c12a137e624439a9f8f519c8afda78 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/29c12a137e624439a9f8f519c8afda78 2024-11-16T14:52:45,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e2311db667634a3f8db9f466b4a04515 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e2311db667634a3f8db9f466b4a04515 2024-11-16T14:52:45,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c602113ffa90444daedb46e04a2983d6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c602113ffa90444daedb46e04a2983d6 2024-11-16T14:52:45,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/400757b47500421e87295e9d1abbd89e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/400757b47500421e87295e9d1abbd89e 2024-11-16T14:52:45,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/4f57d0b05f0d4e38b75e6846e8ceddb0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/4f57d0b05f0d4e38b75e6846e8ceddb0 2024-11-16T14:52:45,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1c638c6f73444d3eb701c9a4fe44fd14 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1c638c6f73444d3eb701c9a4fe44fd14 2024-11-16T14:52:45,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e7953bb07304481d903d8717a9f1a9a9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/e7953bb07304481d903d8717a9f1a9a9 2024-11-16T14:52:45,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7e72509d5da1499f80c944655e927551 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7e72509d5da1499f80c944655e927551 2024-11-16T14:52:45,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5557f5bd5d504fc491c1be20fd1658e9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5557f5bd5d504fc491c1be20fd1658e9 2024-11-16T14:52:45,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1a19db89278b497184636b526bb7e0cb to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/1a19db89278b497184636b526bb7e0cb 2024-11-16T14:52:45,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7ae9b81771e34aee88fdd6175a1a7c3d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7ae9b81771e34aee88fdd6175a1a7c3d 2024-11-16T14:52:45,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b9ebe63117a64969b1757d9d0848dc94 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b9ebe63117a64969b1757d9d0848dc94 2024-11-16T14:52:45,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b2cfd753900f40cb849b43ed52a0d2d1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b2cfd753900f40cb849b43ed52a0d2d1 2024-11-16T14:52:45,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c0f4793ef540474eb982311e1c4c3b3c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/c0f4793ef540474eb982311e1c4c3b3c 2024-11-16T14:52:45,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b6922e3ed468410896d55477f2dad08e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/b6922e3ed468410896d55477f2dad08e 2024-11-16T14:52:45,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/967be0089da142ec90457d1fa11ebf83 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/967be0089da142ec90457d1fa11ebf83 2024-11-16T14:52:45,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/480bb16617284398ad2f2efd3ba66174 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/480bb16617284398ad2f2efd3ba66174 2024-11-16T14:52:45,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/2be171c401aa4f379557e459e632916c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/2be171c401aa4f379557e459e632916c 2024-11-16T14:52:45,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5b59852571ed4bf9b9f72a88826c11e9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/5b59852571ed4bf9b9f72a88826c11e9 2024-11-16T14:52:45,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/a1e854ca363e434bb8e8786588dc0fec to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/a1e854ca363e434bb8e8786588dc0fec 2024-11-16T14:52:45,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/af7beeb455c14e9b93ebcdff25b6730b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/af7beeb455c14e9b93ebcdff25b6730b 2024-11-16T14:52:45,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/42d40bf41697425d8ea6b54f287b1088 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/42d40bf41697425d8ea6b54f287b1088 2024-11-16T14:52:45,165 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/recovered.edits/417.seqid, newMaxSeqId=417, maxSeqId=1 2024-11-16T14:52:45,165 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. 2024-11-16T14:52:45,165 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 0aa24bef827bbc1122cec8569b1d4491: Waiting for close lock at 1731768763811Running coprocessor pre-close hooks at 1731768763811Disabling compacts and flushes for region at 1731768763811Disabling writes for close at 1731768763811Obtaining lock to block concurrent updates at 1731768763811Preparing flush snapshotting stores in 0aa24bef827bbc1122cec8569b1d4491 at 1731768763811Finished memstore snapshotting TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731768763812 (+1 ms)Flushing stores of TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491. at 1731768763813 (+1 ms)Flushing 0aa24bef827bbc1122cec8569b1d4491/A: creating writer at 1731768763813Flushing 0aa24bef827bbc1122cec8569b1d4491/A: appending metadata at 1731768763819 (+6 ms)Flushing 0aa24bef827bbc1122cec8569b1d4491/A: closing flushed file at 1731768763819Flushing 0aa24bef827bbc1122cec8569b1d4491/B: creating writer at 1731768764227 (+408 ms)Flushing 0aa24bef827bbc1122cec8569b1d4491/B: appending metadata at 1731768764229 (+2 ms)Flushing 0aa24bef827bbc1122cec8569b1d4491/B: closing flushed file at 1731768764229Flushing 0aa24bef827bbc1122cec8569b1d4491/C: creating writer at 1731768764640 (+411 ms)Flushing 0aa24bef827bbc1122cec8569b1d4491/C: appending metadata at 1731768764644 (+4 ms)Flushing 0aa24bef827bbc1122cec8569b1d4491/C: closing flushed file at 1731768764644Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e0dbc9f: reopening flushed file at 1731768765052 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2536e0d8: reopening flushed file at 1731768765056 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4da3aa1c: reopening flushed file at 1731768765060 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0aa24bef827bbc1122cec8569b1d4491 in 1254ms, sequenceid=414, compaction requested=true at 1731768765065 (+5 ms)Writing region close event to WAL at 1731768765162 (+97 ms)Running coprocessor post-close hooks at 1731768765165 (+3 ms)Closed at 1731768765165 2024-11-16T14:52:45,167 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:45,168 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=0aa24bef827bbc1122cec8569b1d4491, regionState=CLOSED 2024-11-16T14:52:45,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:52:45,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-16T14:52:45,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 0aa24bef827bbc1122cec8569b1d4491, server=fc3fda29f660,43955,1731768673386 in 1.5120 sec 2024-11-16T14:52:45,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-16T14:52:45,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0aa24bef827bbc1122cec8569b1d4491, UNASSIGN in 1.5170 sec 2024-11-16T14:52:45,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-16T14:52:45,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5210 sec 2024-11-16T14:52:45,176 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768765176"}]},"ts":"1731768765176"} 2024-11-16T14:52:45,177 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:52:45,177 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:52:45,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5350 sec 2024-11-16T14:52:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T14:52:45,790 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:45,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:52:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:45,793 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T14:52:45,794 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:45,795 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:45,797 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/recovered.edits] 2024-11-16T14:52:45,799 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/4109d421fa774276ab8d4da19b0f362f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/4109d421fa774276ab8d4da19b0f362f 2024-11-16T14:52:45,801 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/79d8c7bfcf7e4cabac9131e60267b818 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/79d8c7bfcf7e4cabac9131e60267b818 2024-11-16T14:52:45,801 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/8add71eb9d76457f93348188b9e2e5c0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/8add71eb9d76457f93348188b9e2e5c0 2024-11-16T14:52:45,802 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/c0040dd042a446e98c7d66f465fb1f09 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/A/c0040dd042a446e98c7d66f465fb1f09 2024-11-16T14:52:45,804 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/58a04e5077494901ba31ab35a9db5840 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/58a04e5077494901ba31ab35a9db5840 2024-11-16T14:52:45,805 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/76aabc3447a94f9f9633b7414b433440 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/76aabc3447a94f9f9633b7414b433440 2024-11-16T14:52:45,806 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/f12793f6af6a415a8d767d589af38ad9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/f12793f6af6a415a8d767d589af38ad9 2024-11-16T14:52:45,807 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/fa4ef75865194ae794b435ee6bde58b7 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/B/fa4ef75865194ae794b435ee6bde58b7 2024-11-16T14:52:45,809 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7fbcb29d38ad495cbe08762ed7bef861 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/7fbcb29d38ad495cbe08762ed7bef861 2024-11-16T14:52:45,810 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/9dc07c447c7a406d91ffdcf50f3ea42e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/9dc07c447c7a406d91ffdcf50f3ea42e 2024-11-16T14:52:45,811 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/d6db674b45384ca2903dbf9a41dcbbaf to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/d6db674b45384ca2903dbf9a41dcbbaf 2024-11-16T14:52:45,811 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f2d2442a461e4e0fb961677865010950 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/C/f2d2442a461e4e0fb961677865010950 2024-11-16T14:52:45,813 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/recovered.edits/417.seqid to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491/recovered.edits/417.seqid 2024-11-16T14:52:45,814 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/0aa24bef827bbc1122cec8569b1d4491 2024-11-16T14:52:45,814 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:52:45,816 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:45,819 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:52:45,821 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:52:45,822 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:45,822 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:52:45,822 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731768765822"}]},"ts":"9223372036854775807"} 2024-11-16T14:52:45,824 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:52:45,824 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0aa24bef827bbc1122cec8569b1d4491, NAME => 'TestAcidGuarantees,,1731768737418.0aa24bef827bbc1122cec8569b1d4491.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:52:45,824 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:52:45,824 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731768765824"}]},"ts":"9223372036854775807"} 2024-11-16T14:52:45,826 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:52:45,827 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:45,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-11-16T14:52:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T14:52:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:52:45,901 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:45,909 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1102405766_22 at /127.0.0.1:35178 [Waiting for operation #221] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1737428431_22 at /127.0.0.1:48348 [Waiting for operation #354] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=460 (was 373) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4845 (was 5062) 2024-11-16T14:52:45,916 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=460, ProcessCount=11, AvailableMemoryMB=4845 2024-11-16T14:52:45,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:52:45,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:52:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:45,920 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:52:45,921 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:45,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-16T14:52:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:52:45,921 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:52:45,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742139_1315 (size=1061) 2024-11-16T14:52:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:52:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:52:46,329 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d 2024-11-16T14:52:46,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742140_1316 (size=53) 2024-11-16T14:52:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:52:46,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:52:46,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 49b096ff4ffb696a771be33a33a5e082, disabling compactions & flushes 2024-11-16T14:52:46,735 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:46,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:46,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. after waiting 0 ms 2024-11-16T14:52:46,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:46,735 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:46,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 49b096ff4ffb696a771be33a33a5e082: Waiting for close lock at 1731768766735Disabling compacts and flushes for region at 1731768766735Disabling writes for close at 1731768766735Writing region close event to WAL at 1731768766735Closed at 1731768766735 2024-11-16T14:52:46,737 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:52:46,738 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731768766737"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731768766737"}]},"ts":"1731768766737"} 2024-11-16T14:52:46,740 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:52:46,741 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:52:46,742 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768766741"}]},"ts":"1731768766741"} 2024-11-16T14:52:46,743 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:52:46,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, ASSIGN}] 2024-11-16T14:52:46,745 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, ASSIGN 2024-11-16T14:52:46,746 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, ASSIGN; state=OFFLINE, location=fc3fda29f660,43955,1731768673386; forceNewPlan=false, retain=false 2024-11-16T14:52:46,897 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=49b096ff4ffb696a771be33a33a5e082, regionState=OPENING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:46,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, ASSIGN because future has completed 2024-11-16T14:52:46,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:52:47,056 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:47,056 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:52:47,057 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,057 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:52:47,057 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,057 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,058 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,059 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:52:47,059 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49b096ff4ffb696a771be33a33a5e082 columnFamilyName A 2024-11-16T14:52:47,059 DEBUG [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:47,059 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(327): Store=49b096ff4ffb696a771be33a33a5e082/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:52:47,060 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,060 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:52:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:52:47,061 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49b096ff4ffb696a771be33a33a5e082 columnFamilyName B 2024-11-16T14:52:47,061 DEBUG [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:47,061 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(327): Store=49b096ff4ffb696a771be33a33a5e082/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:52:47,061 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,062 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:52:47,062 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49b096ff4ffb696a771be33a33a5e082 columnFamilyName C 2024-11-16T14:52:47,062 DEBUG [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:47,062 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(327): Store=49b096ff4ffb696a771be33a33a5e082/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:52:47,063 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:47,063 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,063 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,064 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,065 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,065 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,065 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:52:47,067 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,069 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:52:47,069 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 49b096ff4ffb696a771be33a33a5e082; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59489557, jitterRate=-0.1135365217924118}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:52:47,069 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:47,070 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 49b096ff4ffb696a771be33a33a5e082: Running coprocessor pre-open hook at 1731768767057Writing region info on filesystem at 1731768767057Initializing all the Stores at 1731768767057Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768767057Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768767058 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768767058Cleaning up temporary data from old regions at 1731768767065 (+7 ms)Running coprocessor post-open hooks at 1731768767069 (+4 ms)Region opened successfully at 1731768767070 (+1 ms) 2024-11-16T14:52:47,071 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., pid=89, masterSystemTime=1731768767054 2024-11-16T14:52:47,072 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:47,072 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:47,073 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=49b096ff4ffb696a771be33a33a5e082, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:47,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:52:47,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-16T14:52:47,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 in 174 msec 2024-11-16T14:52:47,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-16T14:52:47,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, ASSIGN in 332 msec 2024-11-16T14:52:47,078 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:52:47,078 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768767078"}]},"ts":"1731768767078"} 2024-11-16T14:52:47,079 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:52:47,080 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:52:47,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-16T14:52:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-16T14:52:48,071 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:48,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59596ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:48,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:48,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:48,073 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0e0570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:48,075 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cbde95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:48,076 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:48,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:48,080 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:52:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fc3fda29f660,35823,1731768672605 2024-11-16T14:52:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a089cc0 2024-11-16T14:52:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:52:48,084 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:52:48,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:52:48,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:52:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T14:52:48,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742141_1317 (size=1097) 2024-11-16T14:52:48,506 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T14:52:48,506 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T14:52:48,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:52:48,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, REOPEN/MOVE}] 2024-11-16T14:52:48,512 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, REOPEN/MOVE 2024-11-16T14:52:48,513 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=49b096ff4ffb696a771be33a33a5e082, regionState=CLOSING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:48,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, REOPEN/MOVE because future has completed 2024-11-16T14:52:48,515 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:52:48,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:52:48,667 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,667 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:52:48,667 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 49b096ff4ffb696a771be33a33a5e082, disabling compactions & flushes 2024-11-16T14:52:48,667 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:48,667 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:48,667 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. after waiting 0 ms 2024-11-16T14:52:48,667 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:48,671 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T14:52:48,672 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:48,672 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 49b096ff4ffb696a771be33a33a5e082: Waiting for close lock at 1731768768667Running coprocessor pre-close hooks at 1731768768667Disabling compacts and flushes for region at 1731768768667Disabling writes for close at 1731768768667Writing region close event to WAL at 1731768768668 (+1 ms)Running coprocessor post-close hooks at 1731768768671 (+3 ms)Closed at 1731768768672 (+1 ms) 2024-11-16T14:52:48,672 WARN [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 49b096ff4ffb696a771be33a33a5e082 to self. 2024-11-16T14:52:48,674 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,675 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=49b096ff4ffb696a771be33a33a5e082, regionState=CLOSED 2024-11-16T14:52:48,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:52:48,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-16T14:52:48,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 in 162 msec 2024-11-16T14:52:48,679 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, REOPEN/MOVE; state=CLOSED, location=fc3fda29f660,43955,1731768673386; forceNewPlan=false, retain=true 2024-11-16T14:52:48,830 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=49b096ff4ffb696a771be33a33a5e082, regionState=OPENING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:48,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, REOPEN/MOVE because future has completed 2024-11-16T14:52:48,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:52:48,987 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:48,987 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:52:48,987 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,987 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:52:48,987 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,987 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,989 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,989 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:52:48,990 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49b096ff4ffb696a771be33a33a5e082 columnFamilyName A 2024-11-16T14:52:48,991 DEBUG [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:48,991 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(327): Store=49b096ff4ffb696a771be33a33a5e082/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:52:48,992 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,992 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:52:48,992 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49b096ff4ffb696a771be33a33a5e082 columnFamilyName B 2024-11-16T14:52:48,992 DEBUG [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:48,993 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(327): Store=49b096ff4ffb696a771be33a33a5e082/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:52:48,993 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,993 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:52:48,994 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49b096ff4ffb696a771be33a33a5e082 columnFamilyName C 2024-11-16T14:52:48,994 DEBUG [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:48,994 INFO [StoreOpener-49b096ff4ffb696a771be33a33a5e082-1 {}] regionserver.HStore(327): Store=49b096ff4ffb696a771be33a33a5e082/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:52:48,994 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:48,994 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,995 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,996 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,997 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,997 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,997 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:52:48,999 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:48,999 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 49b096ff4ffb696a771be33a33a5e082; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72690399, jitterRate=0.08317135274410248}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:52:48,999 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:49,000 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 49b096ff4ffb696a771be33a33a5e082: Running coprocessor pre-open hook at 1731768768988Writing region info on filesystem at 1731768768988Initializing all the Stores at 1731768768988Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768768988Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768768988Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768768988Cleaning up temporary data from old regions at 1731768768997 (+9 ms)Running coprocessor post-open hooks at 1731768768999 (+2 ms)Region opened successfully at 1731768769000 (+1 ms) 2024-11-16T14:52:49,001 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., pid=94, masterSystemTime=1731768768984 2024-11-16T14:52:49,002 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,002 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,002 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=49b096ff4ffb696a771be33a33a5e082, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:52:49,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-16T14:52:49,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 in 173 msec 2024-11-16T14:52:49,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-16T14:52:49,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, REOPEN/MOVE in 496 msec 2024-11-16T14:52:49,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-16T14:52:49,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 501 msec 2024-11-16T14:52:49,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 926 msec 2024-11-16T14:52:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-16T14:52:49,015 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:49,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4853c07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:49,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:49,017 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb0cdf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:49,019 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7278aadb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f7d3c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:49,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:49,021 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5196e3ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:49,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5abb8d0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48b614aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:49,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:49,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f99ca82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:49,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ebd2cd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c87087, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:49,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:49,028 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b8706c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:49,030 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:49,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69155f6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7679e723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:49,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:49,032 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b2870a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:49,034 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68d56088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f259004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:49,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:49,038 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67431b46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:49,039 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30969690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526ed020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:49,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:49,042 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111dbc50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:49,044 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c700ee7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2ec5f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:49,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:49,047 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b178db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:49,048 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2938a7e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ee78e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:49,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:49,051 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4072cdb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:49,053 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:49,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f3ace6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ee060c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,063 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:52:49,063 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:52:49,064 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:52:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:52:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:52:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71735d19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:52:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:52:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:52:49,067 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:52:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ead04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:52:49,069 DEBUG [Thread-1463 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:49,069 DEBUG [Thread-1457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:49,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:49,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:52:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:52:49,079 DEBUG [Thread-1470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:49,080 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:49,080 DEBUG [Thread-1476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-16T14:52:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:49,082 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:52:49,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:49,083 DEBUG [Thread-1465 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:49,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:49,083 DEBUG [Thread-1474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:49,083 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:49,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:49,084 DEBUG [Thread-1474 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:49,084 DEBUG [Thread-1474 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:52:49,086 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:52:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:49,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:52:49,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:52:49,091 DEBUG [Thread-1461 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:49,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:52:49,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:49,095 DEBUG [Thread-1459 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:49,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:52:49,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:49,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:52:49,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:49,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:52:49,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:52:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:49,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:52:49,102 DEBUG [Thread-1468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:49,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:52:49,115 DEBUG [Thread-1472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:52:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:52:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:52:49,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:52:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:52:49,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731768769121, max timestamp=1731768769144Num uniques 859; 2024-11-16T14:52:49,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:52:49,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731768769121, max timestamp=1731768769144Num uniques 859; 2024-11-16T14:52:49,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:52:49,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768769121, max timestamp=1731768769144Num uniques 859; 2024-11-16T14:52:49,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:52:49,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fae297812e18487f8e17a75e9be191f3_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768769083/Put/seqid=0 2024-11-16T14:52:49,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742142_1318 (size=12154) 2024-11-16T14:52:49,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768829184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:52:49,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768829186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768829187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768829189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768829189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:52:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768829300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768829311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768829311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768829311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768829311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:52:49,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:49,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:52:49,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768829521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768829521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768829521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768829521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768829521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:52:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,586 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:49,590 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fae297812e18487f8e17a75e9be191f3_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fae297812e18487f8e17a75e9be191f3_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:49,591 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/ffc86e6d17f343b29d33a97450dc23f9, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:49,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/ffc86e6d17f343b29d33a97450dc23f9 is 175, key is test_row_0/A:col10/1731768769083/Put/seqid=0 2024-11-16T14:52:49,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742143_1319 (size=30955) 2024-11-16T14:52:49,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/ffc86e6d17f343b29d33a97450dc23f9 2024-11-16T14:52:49,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/eb7bfa49003245dc9dbff3b227c21133 is 50, key is test_row_0/B:col10/1731768769083/Put/seqid=0 2024-11-16T14:52:49,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742144_1320 (size=12001) 2024-11-16T14:52:49,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:52:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:52:49,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768829831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768829831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768829831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768829831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768829831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:49,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:52:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:49,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:52:50,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:50,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:50,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:50,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/eb7bfa49003245dc9dbff3b227c21133 2024-11-16T14:52:50,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/53116770e74d496e9a6021a6e00cdb09 is 50, key is test_row_0/C:col10/1731768769083/Put/seqid=0 2024-11-16T14:52:50,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742145_1321 (size=12001) 2024-11-16T14:52:50,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:52:50,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:50,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:50,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:50,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:52:50,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:52:50,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:50,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:50,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:50,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:50,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768830340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:50,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768830340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:50,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768830340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:50,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768830342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:50,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:50,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768830351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:50,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:52:50,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/53116770e74d496e9a6021a6e00cdb09 2024-11-16T14:52:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:50,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:50,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/ffc86e6d17f343b29d33a97450dc23f9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/ffc86e6d17f343b29d33a97450dc23f9 2024-11-16T14:52:50,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/ffc86e6d17f343b29d33a97450dc23f9, entries=150, sequenceid=15, filesize=30.2 K 2024-11-16T14:52:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/eb7bfa49003245dc9dbff3b227c21133 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/eb7bfa49003245dc9dbff3b227c21133 2024-11-16T14:52:50,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/eb7bfa49003245dc9dbff3b227c21133, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T14:52:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/53116770e74d496e9a6021a6e00cdb09 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/53116770e74d496e9a6021a6e00cdb09 2024-11-16T14:52:50,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/53116770e74d496e9a6021a6e00cdb09, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T14:52:50,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 49b096ff4ffb696a771be33a33a5e082 in 1397ms, sequenceid=15, compaction requested=false 2024-11-16T14:52:50,489 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T14:52:50,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:50,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T14:52:50,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:50,621 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T14:52:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:52:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:52:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:52:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:50,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ca4b259a5a674599bc6122f6bcca25d4_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768769162/Put/seqid=0 2024-11-16T14:52:50,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742146_1322 (size=12154) 2024-11-16T14:52:50,689 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:52:51,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:51,054 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ca4b259a5a674599bc6122f6bcca25d4_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ca4b259a5a674599bc6122f6bcca25d4_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:51,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/5a2574b4723d4c9796cc1f7e050dbf4d, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:51,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/5a2574b4723d4c9796cc1f7e050dbf4d is 175, key is test_row_0/A:col10/1731768769162/Put/seqid=0 2024-11-16T14:52:51,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742147_1323 (size=30955) 2024-11-16T14:52:51,065 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/5a2574b4723d4c9796cc1f7e050dbf4d 2024-11-16T14:52:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/0b7e33ce84054a10a0ba9bebf3a42e45 is 50, key is test_row_0/B:col10/1731768769162/Put/seqid=0 2024-11-16T14:52:51,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742148_1324 (size=12001) 2024-11-16T14:52:51,102 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/0b7e33ce84054a10a0ba9bebf3a42e45 2024-11-16T14:52:51,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/962150132adb42f59d5a151e5d80f264 is 50, key is test_row_0/C:col10/1731768769162/Put/seqid=0 2024-11-16T14:52:51,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742149_1325 (size=12001) 2024-11-16T14:52:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:52:51,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:51,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768831373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768831380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768831381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768831388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768831388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768831500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768831500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768831500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768831500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768831500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,542 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/962150132adb42f59d5a151e5d80f264 2024-11-16T14:52:51,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/5a2574b4723d4c9796cc1f7e050dbf4d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/5a2574b4723d4c9796cc1f7e050dbf4d 2024-11-16T14:52:51,557 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/5a2574b4723d4c9796cc1f7e050dbf4d, entries=150, sequenceid=50, filesize=30.2 K 2024-11-16T14:52:51,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/0b7e33ce84054a10a0ba9bebf3a42e45 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/0b7e33ce84054a10a0ba9bebf3a42e45 2024-11-16T14:52:51,562 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/0b7e33ce84054a10a0ba9bebf3a42e45, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T14:52:51,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/962150132adb42f59d5a151e5d80f264 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/962150132adb42f59d5a151e5d80f264 2024-11-16T14:52:51,567 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/962150132adb42f59d5a151e5d80f264, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T14:52:51,568 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 49b096ff4ffb696a771be33a33a5e082 in 948ms, sequenceid=50, compaction requested=false 2024-11-16T14:52:51,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:51,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:51,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-16T14:52:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-16T14:52:51,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-16T14:52:51,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4870 sec 2024-11-16T14:52:51,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.4950 sec 2024-11-16T14:52:51,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:52:51,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:52:51,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:51,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:52:51,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:51,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:52:51,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:51,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164dfdaf4a126246bcbb9b9d4e956a2e26_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768771379/Put/seqid=0 2024-11-16T14:52:51,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742150_1326 (size=14594) 2024-11-16T14:52:51,733 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:51,738 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164dfdaf4a126246bcbb9b9d4e956a2e26_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164dfdaf4a126246bcbb9b9d4e956a2e26_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:51,738 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/29abd38a6c074d0396592de5b1419478, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:51,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/29abd38a6c074d0396592de5b1419478 is 175, key is test_row_0/A:col10/1731768771379/Put/seqid=0 2024-11-16T14:52:51,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742151_1327 (size=39549) 2024-11-16T14:52:51,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768831759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768831761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768831762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768831764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768831764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768831870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768831881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768831881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768831881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768831881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768832090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768832100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768832100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768832101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768832101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,147 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/29abd38a6c074d0396592de5b1419478 2024-11-16T14:52:52,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/c4bca69742804ee9aa45b39606eb15b2 is 50, key is test_row_0/B:col10/1731768771379/Put/seqid=0 2024-11-16T14:52:52,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742152_1328 (size=12001) 2024-11-16T14:52:52,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/c4bca69742804ee9aa45b39606eb15b2 2024-11-16T14:52:52,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/f5769305bf994f1bbd8ed1a1834e222b is 50, key is test_row_0/C:col10/1731768771379/Put/seqid=0 2024-11-16T14:52:52,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742153_1329 (size=12001) 2024-11-16T14:52:52,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/f5769305bf994f1bbd8ed1a1834e222b 2024-11-16T14:52:52,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/29abd38a6c074d0396592de5b1419478 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/29abd38a6c074d0396592de5b1419478 2024-11-16T14:52:52,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/29abd38a6c074d0396592de5b1419478, entries=200, sequenceid=62, filesize=38.6 K 2024-11-16T14:52:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/c4bca69742804ee9aa45b39606eb15b2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c4bca69742804ee9aa45b39606eb15b2 2024-11-16T14:52:52,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c4bca69742804ee9aa45b39606eb15b2, entries=150, sequenceid=62, filesize=11.7 K 2024-11-16T14:52:52,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/f5769305bf994f1bbd8ed1a1834e222b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f5769305bf994f1bbd8ed1a1834e222b 2024-11-16T14:52:52,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f5769305bf994f1bbd8ed1a1834e222b, entries=150, sequenceid=62, filesize=11.7 K 2024-11-16T14:52:52,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 49b096ff4ffb696a771be33a33a5e082 in 583ms, sequenceid=62, compaction requested=true 2024-11-16T14:52:52,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:52,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:52,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:52,300 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:52,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:52,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:52,300 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:52,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:52,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:52,304 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:52,304 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:52,304 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/A is initiating minor compaction (all files) 2024-11-16T14:52:52,304 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/B is initiating minor compaction (all files) 2024-11-16T14:52:52,304 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/A in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:52,304 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/B in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:52,304 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/eb7bfa49003245dc9dbff3b227c21133, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/0b7e33ce84054a10a0ba9bebf3a42e45, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c4bca69742804ee9aa45b39606eb15b2] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=35.2 K 2024-11-16T14:52:52,304 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/ffc86e6d17f343b29d33a97450dc23f9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/5a2574b4723d4c9796cc1f7e050dbf4d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/29abd38a6c074d0396592de5b1419478] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=99.1 K 2024-11-16T14:52:52,304 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:52,304 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/ffc86e6d17f343b29d33a97450dc23f9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/5a2574b4723d4c9796cc1f7e050dbf4d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/29abd38a6c074d0396592de5b1419478] 2024-11-16T14:52:52,305 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb7bfa49003245dc9dbff3b227c21133, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731768769075 2024-11-16T14:52:52,305 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ffc86e6d17f343b29d33a97450dc23f9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731768769075 2024-11-16T14:52:52,305 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b7e33ce84054a10a0ba9bebf3a42e45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731768769162 2024-11-16T14:52:52,305 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a2574b4723d4c9796cc1f7e050dbf4d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731768769162 2024-11-16T14:52:52,305 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4bca69742804ee9aa45b39606eb15b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731768771377 2024-11-16T14:52:52,305 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 29abd38a6c074d0396592de5b1419478, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731768771377 2024-11-16T14:52:52,319 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:52,320 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/e6b867d8e97e4460aae743085c35dccd is 50, key is test_row_0/B:col10/1731768771379/Put/seqid=0 2024-11-16T14:52:52,331 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:52,346 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116eb6843ee445a4a7ba2aa5a7bed98ab6b_49b096ff4ffb696a771be33a33a5e082 store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:52,348 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116eb6843ee445a4a7ba2aa5a7bed98ab6b_49b096ff4ffb696a771be33a33a5e082, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:52,348 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eb6843ee445a4a7ba2aa5a7bed98ab6b_49b096ff4ffb696a771be33a33a5e082 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:52,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742154_1330 (size=12104) 2024-11-16T14:52:52,357 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/e6b867d8e97e4460aae743085c35dccd as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/e6b867d8e97e4460aae743085c35dccd 2024-11-16T14:52:52,362 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/B of 49b096ff4ffb696a771be33a33a5e082 into e6b867d8e97e4460aae743085c35dccd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:52,362 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:52,362 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/B, priority=13, startTime=1731768772300; duration=0sec 2024-11-16T14:52:52,362 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:52,362 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:B 2024-11-16T14:52:52,362 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:52,363 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:52,363 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/C is initiating minor compaction (all files) 2024-11-16T14:52:52,363 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/C in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:52,363 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/53116770e74d496e9a6021a6e00cdb09, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/962150132adb42f59d5a151e5d80f264, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f5769305bf994f1bbd8ed1a1834e222b] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=35.2 K 2024-11-16T14:52:52,364 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53116770e74d496e9a6021a6e00cdb09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731768769075 2024-11-16T14:52:52,364 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 962150132adb42f59d5a151e5d80f264, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731768769162 2024-11-16T14:52:52,364 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5769305bf994f1bbd8ed1a1834e222b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731768771377 2024-11-16T14:52:52,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742155_1331 (size=4469) 2024-11-16T14:52:52,369 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:52,370 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/790543cc33be4e2fa511743df399864e is 50, key is test_row_0/C:col10/1731768771379/Put/seqid=0 2024-11-16T14:52:52,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T14:52:52,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:52:52,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:52,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:52:52,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:52,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:52:52,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:52,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742156_1332 (size=12104) 2024-11-16T14:52:52,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165cfd5a595cf44440b513907191819d50_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768771762/Put/seqid=0 2024-11-16T14:52:52,418 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/790543cc33be4e2fa511743df399864e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/790543cc33be4e2fa511743df399864e 2024-11-16T14:52:52,425 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/C of 49b096ff4ffb696a771be33a33a5e082 into 790543cc33be4e2fa511743df399864e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:52,425 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:52,425 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/C, priority=13, startTime=1731768772300; duration=0sec 2024-11-16T14:52:52,425 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:52,425 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:C 2024-11-16T14:52:52,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742157_1333 (size=14594) 2024-11-16T14:52:52,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768832427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768832429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768832435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768832435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768832445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768832551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768832551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768832551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768832551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768832560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,770 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#A#compaction#283 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:52,770 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/a790b3afee494a7197ad7c6ff5827678 is 175, key is test_row_0/A:col10/1731768771379/Put/seqid=0 2024-11-16T14:52:52,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768832770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768832771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768832771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768832772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742158_1334 (size=31058) 2024-11-16T14:52:52,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768832781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:52,789 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/a790b3afee494a7197ad7c6ff5827678 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a790b3afee494a7197ad7c6ff5827678 2024-11-16T14:52:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:52,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:52,800 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/A of 49b096ff4ffb696a771be33a33a5e082 into a790b3afee494a7197ad7c6ff5827678(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:52,800 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:52,800 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/A, priority=13, startTime=1731768772300; duration=0sec 2024-11-16T14:52:52,800 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:52,800 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:A 2024-11-16T14:52:52,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:52,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165cfd5a595cf44440b513907191819d50_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165cfd5a595cf44440b513907191819d50_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:52,848 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/9753176bd923451fb947c66d58b6097f, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:52,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/9753176bd923451fb947c66d58b6097f is 175, key is test_row_0/A:col10/1731768771762/Put/seqid=0 2024-11-16T14:52:52,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742159_1335 (size=39549) 2024-11-16T14:52:52,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/9753176bd923451fb947c66d58b6097f 2024-11-16T14:52:52,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/f430c2d721a94701b882438c6ebe45be is 50, key is test_row_0/B:col10/1731768771762/Put/seqid=0 2024-11-16T14:52:52,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742160_1336 (size=12001) 2024-11-16T14:52:52,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/f430c2d721a94701b882438c6ebe45be 2024-11-16T14:52:52,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/f33dabf11d6d464682c09c3c4ce91b11 is 50, key is test_row_0/C:col10/1731768771762/Put/seqid=0 2024-11-16T14:52:52,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742161_1337 (size=12001) 2024-11-16T14:52:53,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768833081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768833081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768833081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768833081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768833100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T14:52:53,241 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:53,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-16T14:52:53,243 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:53,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:52:53,244 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:53,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:53,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/f33dabf11d6d464682c09c3c4ce91b11 2024-11-16T14:52:53,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/9753176bd923451fb947c66d58b6097f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/9753176bd923451fb947c66d58b6097f 2024-11-16T14:52:53,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/9753176bd923451fb947c66d58b6097f, entries=200, sequenceid=84, filesize=38.6 K 2024-11-16T14:52:53,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/f430c2d721a94701b882438c6ebe45be as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f430c2d721a94701b882438c6ebe45be 2024-11-16T14:52:53,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f430c2d721a94701b882438c6ebe45be, entries=150, sequenceid=84, filesize=11.7 K 2024-11-16T14:52:53,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/f33dabf11d6d464682c09c3c4ce91b11 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f33dabf11d6d464682c09c3c4ce91b11 2024-11-16T14:52:53,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f33dabf11d6d464682c09c3c4ce91b11, entries=150, sequenceid=84, filesize=11.7 K 2024-11-16T14:52:53,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 49b096ff4ffb696a771be33a33a5e082 in 932ms, sequenceid=84, compaction requested=false 2024-11-16T14:52:53,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:52:53,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T14:52:53,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:53,400 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:52:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:52:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:52:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:52:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:53,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b901896062f94aa982784ea9e497ecfa_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768772443/Put/seqid=0 2024-11-16T14:52:53,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742162_1338 (size=12154) 2024-11-16T14:52:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:52:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:53,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:53,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731768773623, max timestamp=1731768773651Num uniques 859; 2024-11-16T14:52:53,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-16T14:52:53,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768773630, max timestamp=1731768773651Num uniques 859; 2024-11-16T14:52:53,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768773630, max timestamp=1731768773651Num uniques 859; 2024-11-16T14:52:53,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:53,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:53,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768833688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768833704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768833706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768833704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768833706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768833810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:53,839 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b901896062f94aa982784ea9e497ecfa_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b901896062f94aa982784ea9e497ecfa_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/c9fb0a064b3d4539ab5a66dd297529f4, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:53,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/c9fb0a064b3d4539ab5a66dd297529f4 is 175, key is test_row_0/A:col10/1731768772443/Put/seqid=0 2024-11-16T14:52:53,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768833831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768833831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768833831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768833831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:53,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742163_1339 (size=30955) 2024-11-16T14:52:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:52:54,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768834031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768834050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768834050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768834050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768834050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,246 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/c9fb0a064b3d4539ab5a66dd297529f4 2024-11-16T14:52:54,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/3a6ca68ff3974f01abdd675c4d9dcbc4 is 50, key is test_row_0/B:col10/1731768772443/Put/seqid=0 2024-11-16T14:52:54,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742164_1340 (size=12001) 2024-11-16T14:52:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768834341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:52:54,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768834370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768834371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768834371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768834371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,656 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/3a6ca68ff3974f01abdd675c4d9dcbc4 2024-11-16T14:52:54,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/a5c1fcb9000b4094a4366a8c11b1c655 is 50, key is test_row_0/C:col10/1731768772443/Put/seqid=0 2024-11-16T14:52:54,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742165_1341 (size=12001) 2024-11-16T14:52:54,680 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/a5c1fcb9000b4094a4366a8c11b1c655 2024-11-16T14:52:54,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/c9fb0a064b3d4539ab5a66dd297529f4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9fb0a064b3d4539ab5a66dd297529f4 2024-11-16T14:52:54,689 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9fb0a064b3d4539ab5a66dd297529f4, entries=150, sequenceid=97, filesize=30.2 K 2024-11-16T14:52:54,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/3a6ca68ff3974f01abdd675c4d9dcbc4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/3a6ca68ff3974f01abdd675c4d9dcbc4 2024-11-16T14:52:54,695 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/3a6ca68ff3974f01abdd675c4d9dcbc4, entries=150, sequenceid=97, filesize=11.7 K 2024-11-16T14:52:54,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/a5c1fcb9000b4094a4366a8c11b1c655 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a5c1fcb9000b4094a4366a8c11b1c655 2024-11-16T14:52:54,699 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a5c1fcb9000b4094a4366a8c11b1c655, entries=150, sequenceid=97, filesize=11.7 K 2024-11-16T14:52:54,700 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for 49b096ff4ffb696a771be33a33a5e082 in 1300ms, sequenceid=97, compaction requested=true 2024-11-16T14:52:54,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:54,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:54,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-16T14:52:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-16T14:52:54,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-16T14:52:54,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4580 sec 2024-11-16T14:52:54,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.4630 sec 2024-11-16T14:52:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:54,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=143.13 KB heapSize=400.61 KB 2024-11-16T14:52:54,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:52:54,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:54,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:52:54,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:54,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:52:54,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:54,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a6701a20c26a4585930eca543b04a8af_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768773684/Put/seqid=0 2024-11-16T14:52:54,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742166_1342 (size=14744) 2024-11-16T14:52:54,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:54,887 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a6701a20c26a4585930eca543b04a8af_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a6701a20c26a4585930eca543b04a8af_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:54,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/82081788ac9042058f9d48f10428fbca, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:54,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/82081788ac9042058f9d48f10428fbca is 175, key is test_row_0/A:col10/1731768773684/Put/seqid=0 2024-11-16T14:52:54,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742167_1343 (size=39699) 2024-11-16T14:52:54,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768834901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768834901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768834907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768834910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:54,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768834916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768835021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768835030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768835040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768835240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768835251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768835251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,296 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/82081788ac9042058f9d48f10428fbca 2024-11-16T14:52:55,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/5e50a4b2a12241cb903b40022302dad5 is 50, key is test_row_0/B:col10/1731768773684/Put/seqid=0 2024-11-16T14:52:55,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742168_1344 (size=12101) 2024-11-16T14:52:55,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/5e50a4b2a12241cb903b40022302dad5 2024-11-16T14:52:55,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/7bb111f91f804e25850bd5244af6fbcb is 50, key is test_row_0/C:col10/1731768773684/Put/seqid=0 2024-11-16T14:52:55,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742169_1345 (size=12101) 2024-11-16T14:52:55,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/7bb111f91f804e25850bd5244af6fbcb 2024-11-16T14:52:55,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/82081788ac9042058f9d48f10428fbca as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/82081788ac9042058f9d48f10428fbca 2024-11-16T14:52:55,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/82081788ac9042058f9d48f10428fbca, entries=200, sequenceid=133, filesize=38.8 K 2024-11-16T14:52:55,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/5e50a4b2a12241cb903b40022302dad5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5e50a4b2a12241cb903b40022302dad5 2024-11-16T14:52:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5e50a4b2a12241cb903b40022302dad5, entries=150, sequenceid=133, filesize=11.8 K 2024-11-16T14:52:55,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/7bb111f91f804e25850bd5244af6fbcb as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/7bb111f91f804e25850bd5244af6fbcb 2024-11-16T14:52:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/7bb111f91f804e25850bd5244af6fbcb, entries=150, sequenceid=133, filesize=11.8 K 2024-11-16T14:52:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~400.56 KB/410176, currentSize=40.25 KB/41220 for 49b096ff4ffb696a771be33a33a5e082 in 473ms, sequenceid=133, compaction requested=true 2024-11-16T14:52:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:55,343 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:55,343 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,344 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:55,344 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141261 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,344 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/B is initiating minor compaction (all files) 2024-11-16T14:52:55,344 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/A is initiating minor compaction (all files) 2024-11-16T14:52:55,345 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/B in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:55,345 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/e6b867d8e97e4460aae743085c35dccd, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f430c2d721a94701b882438c6ebe45be, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/3a6ca68ff3974f01abdd675c4d9dcbc4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5e50a4b2a12241cb903b40022302dad5] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=47.1 K 2024-11-16T14:52:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,345 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/A in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,345 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a790b3afee494a7197ad7c6ff5827678, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/9753176bd923451fb947c66d58b6097f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9fb0a064b3d4539ab5a66dd297529f4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/82081788ac9042058f9d48f10428fbca] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=138.0 K 2024-11-16T14:52:55,345 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b867d8e97e4460aae743085c35dccd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731768771377 2024-11-16T14:52:55,345 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:55,345 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a790b3afee494a7197ad7c6ff5827678, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/9753176bd923451fb947c66d58b6097f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9fb0a064b3d4539ab5a66dd297529f4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/82081788ac9042058f9d48f10428fbca] 2024-11-16T14:52:55,345 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f430c2d721a94701b882438c6ebe45be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731768771759 2024-11-16T14:52:55,346 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a790b3afee494a7197ad7c6ff5827678, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731768771377 2024-11-16T14:52:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,346 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a6ca68ff3974f01abdd675c4d9dcbc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731768772432 2024-11-16T14:52:55,346 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9753176bd923451fb947c66d58b6097f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731768771759 2024-11-16T14:52:55,346 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e50a4b2a12241cb903b40022302dad5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731768773684 2024-11-16T14:52:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,347 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9fb0a064b3d4539ab5a66dd297529f4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731768772432 2024-11-16T14:52:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,347 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82081788ac9042058f9d48f10428fbca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731768773684 2024-11-16T14:52:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,355 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,355 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/6cb051435916468fbdbb25b1c896d4a2 is 50, key is test_row_0/B:col10/1731768773684/Put/seqid=0 2024-11-16T14:52:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,356 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,371 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411161fc8b27c13ab44c9af529d5cce87550c_49b096ff4ffb696a771be33a33a5e082 store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,373 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411161fc8b27c13ab44c9af529d5cce87550c_49b096ff4ffb696a771be33a33a5e082, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,373 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161fc8b27c13ab44c9af529d5cce87550c_49b096ff4ffb696a771be33a33a5e082 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742170_1346 (size=12341) 2024-11-16T14:52:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T14:52:55,390 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-16T14:52:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,392 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:52:55,393 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742171_1347 (size=4469) 2024-11-16T14:52:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:52:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T14:52:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:55,547 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-16T14:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:52:55,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:55,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:52:55,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:55,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:52:55,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164062ca75380f46ab8ab5836954a81e51_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768774905/Put/seqid=0 2024-11-16T14:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742172_1348 (size=9814) 2024-11-16T14:52:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,579 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164062ca75380f46ab8ab5836954a81e51_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164062ca75380f46ab8ab5836954a81e51_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/c9bff21fdfe84642b43ac4320e31590a, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/c9bff21fdfe84642b43ac4320e31590a is 175, key is test_row_0/A:col10/1731768774905/Put/seqid=0 2024-11-16T14:52:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742173_1349 (size=22461) 2024-11-16T14:52:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:55,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768775680, max timestamp=1731768775700Num uniques 859; 2024-11-16T14:52:55,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768775680, max timestamp=1731768775700Num uniques 859; 2024-11-16T14:52:55,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:55,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:55,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768775680, max timestamp=1731768775700Num uniques 859; 2024-11-16T14:52:55,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:52:55,800 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/6cb051435916468fbdbb25b1c896d4a2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6cb051435916468fbdbb25b1c896d4a2 2024-11-16T14:52:55,806 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#A#compaction#295 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:55,807 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/a4563ebb52cb43a88484d7e4713747ff is 175, key is test_row_0/A:col10/1731768773684/Put/seqid=0 2024-11-16T14:52:55,811 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/B of 49b096ff4ffb696a771be33a33a5e082 into 6cb051435916468fbdbb25b1c896d4a2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:55,811 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:55,811 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/B, priority=12, startTime=1731768775343; duration=0sec 2024-11-16T14:52:55,811 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:55,811 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:B 2024-11-16T14:52:55,812 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:52:55,813 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:52:55,813 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/C is initiating minor compaction (all files) 2024-11-16T14:52:55,813 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/C in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:55,813 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/790543cc33be4e2fa511743df399864e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f33dabf11d6d464682c09c3c4ce91b11, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a5c1fcb9000b4094a4366a8c11b1c655, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/7bb111f91f804e25850bd5244af6fbcb] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=47.1 K 2024-11-16T14:52:55,813 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 790543cc33be4e2fa511743df399864e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731768771377 2024-11-16T14:52:55,813 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f33dabf11d6d464682c09c3c4ce91b11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731768771759 2024-11-16T14:52:55,814 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c1fcb9000b4094a4366a8c11b1c655, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731768772432 2024-11-16T14:52:55,814 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bb111f91f804e25850bd5244af6fbcb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731768773684 2024-11-16T14:52:55,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742174_1350 (size=31295) 2024-11-16T14:52:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768835800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768835803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768835805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,832 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/a4563ebb52cb43a88484d7e4713747ff as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a4563ebb52cb43a88484d7e4713747ff 2024-11-16T14:52:55,840 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/A of 49b096ff4ffb696a771be33a33a5e082 into a4563ebb52cb43a88484d7e4713747ff(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:55,840 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:55,840 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/A, priority=12, startTime=1731768775343; duration=0sec 2024-11-16T14:52:55,840 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:55,840 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:A 2024-11-16T14:52:55,845 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:55,846 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/f2a7e22d505b40128a4ef36e12658f67 is 50, key is test_row_0/C:col10/1731768773684/Put/seqid=0 2024-11-16T14:52:55,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742175_1351 (size=12341) 2024-11-16T14:52:55,874 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/f2a7e22d505b40128a4ef36e12658f67 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f2a7e22d505b40128a4ef36e12658f67 2024-11-16T14:52:55,885 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/C of 49b096ff4ffb696a771be33a33a5e082 into f2a7e22d505b40128a4ef36e12658f67(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:55,885 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:55,885 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/C, priority=12, startTime=1731768775343; duration=0sec 2024-11-16T14:52:55,885 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:55,885 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:C 2024-11-16T14:52:55,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768835920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,921 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in 49b096ff4ffb696a771be33a33a5e082 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768835931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768835931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768835931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768835931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,939 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in 49b096ff4ffb696a771be33a33a5e082 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:52:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:56,011 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/c9bff21fdfe84642b43ac4320e31590a 2024-11-16T14:52:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:52:56,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/2798e09c8f7c43418e456dfe8aedb4b8 is 50, key is test_row_0/B:col10/1731768774905/Put/seqid=0 2024-11-16T14:52:56,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742176_1352 (size=9757) 2024-11-16T14:52:56,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768836140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:56,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768836151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:56,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768836151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768836461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:56,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768836471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:56,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768836471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:56,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:56,511 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/2798e09c8f7c43418e456dfe8aedb4b8 2024-11-16T14:52:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:52:56,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/909d0c2815884013b6e2e3b7428e138e is 50, key is test_row_0/C:col10/1731768774905/Put/seqid=0 2024-11-16T14:52:56,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742177_1353 (size=9757) 2024-11-16T14:52:56,552 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/909d0c2815884013b6e2e3b7428e138e 2024-11-16T14:52:56,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/c9bff21fdfe84642b43ac4320e31590a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9bff21fdfe84642b43ac4320e31590a 2024-11-16T14:52:56,563 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9bff21fdfe84642b43ac4320e31590a, entries=100, sequenceid=142, filesize=21.9 K 2024-11-16T14:52:56,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/2798e09c8f7c43418e456dfe8aedb4b8 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/2798e09c8f7c43418e456dfe8aedb4b8 2024-11-16T14:52:56,568 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/2798e09c8f7c43418e456dfe8aedb4b8, entries=100, sequenceid=142, filesize=9.5 K 2024-11-16T14:52:56,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/909d0c2815884013b6e2e3b7428e138e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/909d0c2815884013b6e2e3b7428e138e 2024-11-16T14:52:56,573 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/909d0c2815884013b6e2e3b7428e138e, entries=100, sequenceid=142, filesize=9.5 K 2024-11-16T14:52:56,574 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 49b096ff4ffb696a771be33a33a5e082 in 1028ms, sequenceid=142, compaction requested=false 2024-11-16T14:52:56,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:56,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:56,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-16T14:52:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-16T14:52:56,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-16T14:52:56,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1820 sec 2024-11-16T14:52:56,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.1860 sec 2024-11-16T14:52:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:56,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731768775757, max timestamp=1731768776987Num uniques 1068; 2024-11-16T14:52:56,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T14:52:56,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731768775757, max timestamp=1731768776987Num uniques 1068; 2024-11-16T14:52:56,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731768775757, max timestamp=1731768776987Num uniques 1068; 2024-11-16T14:52:56,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T14:52:56,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T14:52:56,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-11-16T14:52:56,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:52:56,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:56,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:52:56,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:56,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:52:56,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:56,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116511c49adaed149e7951e63c88db6ed1f_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768776987/Put/seqid=0 2024-11-16T14:52:57,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742178_1354 (size=14794) 2024-11-16T14:52:57,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768777019, max timestamp=1731768777049Num uniques 859; 2024-11-16T14:52:57,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768777019, max timestamp=1731768777049Num uniques 859; 2024-11-16T14:52:57,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:57,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:57,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768777019, max timestamp=1731768777049Num uniques 859; 2024-11-16T14:52:57,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:52:57,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731768777075, max timestamp=1731768777114Num uniques 1068; 2024-11-16T14:52:57,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731768777075, max timestamp=1731768777114Num uniques 1068; 2024-11-16T14:52:57,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-16T14:52:57,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-16T14:52:57,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731768777075, max timestamp=1731768777114Num uniques 1068; 2024-11-16T14:52:57,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-16T14:52:57,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731768777154, max timestamp=1731768777179Num uniques 1127; 2024-11-16T14:52:57,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-16T14:52:57,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731768777154, max timestamp=1731768777179Num uniques 1127; 2024-11-16T14:52:57,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-16T14:52:57,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731768777154, max timestamp=1731768777179Num uniques 1127; 2024-11-16T14:52:57,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-16T14:52:57,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731768777230, max timestamp=1731768777251Num uniques 1136; 2024-11-16T14:52:57,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-16T14:52:57,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731768777235, max timestamp=1731768777251Num uniques 1136; 2024-11-16T14:52:57,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-16T14:52:57,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731768777235, max timestamp=1731768777251Num uniques 1136; 2024-11-16T14:52:57,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-16T14:52:57,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731768777286, max timestamp=1731768777315Num uniques 1045; 2024-11-16T14:52:57,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-16T14:52:57,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731768777286, max timestamp=1731768777315Num uniques 1145; 2024-11-16T14:52:57,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731768777286, max timestamp=1731768777315Num uniques 1045; 2024-11-16T14:52:57,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-16T14:52:57,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-16T14:52:57,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731768777361, max timestamp=1731768777385Num uniques 1154; 2024-11-16T14:52:57,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-11-16T14:52:57,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731768777361, max timestamp=1731768777385Num uniques 1154; 2024-11-16T14:52:57,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-11-16T14:52:57,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731768777361, max timestamp=1731768777385Num uniques 1154; 2024-11-16T14:52:57,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=11630, suffix heap size=147984,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=254 2024-11-16T14:52:57,404 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:57,407 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116511c49adaed149e7951e63c88db6ed1f_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116511c49adaed149e7951e63c88db6ed1f_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:57,408 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/f330dfabbd924377be339e19c45a8f16, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:57,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/f330dfabbd924377be339e19c45a8f16 is 175, key is test_row_0/A:col10/1731768776987/Put/seqid=0 2024-11-16T14:52:57,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742179_1355 (size=39749) 2024-11-16T14:52:57,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731768777431, max timestamp=1731768777460Num uniques 1113; 2024-11-16T14:52:57,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=14328, suffix heap size=143720,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=313 2024-11-16T14:52:57,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731768777431, max timestamp=1731768777460Num uniques 1113; 2024-11-16T14:52:57,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=14328, suffix heap size=143720,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=313 2024-11-16T14:52:57,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731768777445, max timestamp=1731768777460Num uniques 1113; 2024-11-16T14:52:57,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-16T14:52:57,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731768777506, max timestamp=1731768777527Num uniques 1172; 2024-11-16T14:52:57,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=12452, suffix heap size=149856,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=272 2024-11-16T14:52:57,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731768777506, max timestamp=1731768777527Num uniques 1022; 2024-11-16T14:52:57,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731768777506, max timestamp=1731768777527Num uniques 1172; 2024-11-16T14:52:57,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-16T14:52:57,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=12452, suffix heap size=149856,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=272 2024-11-16T14:52:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T14:52:57,551 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:52:57,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:52:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-16T14:52:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:52:57,554 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:52:57,554 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:52:57,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:52:57,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731768777583, max timestamp=1731768777602Num uniques 1131; 2024-11-16T14:52:57,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731768777583, max timestamp=1731768777602Num uniques 1131; 2024-11-16T14:52:57,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:57,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-16T14:52:57,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-16T14:52:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768837600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:57,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731768777583, max timestamp=1731768777602Num uniques 1131; 2024-11-16T14:52:57,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-16T14:52:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:52:57,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731768777629, max timestamp=1731768777693Num uniques 1040; 2024-11-16T14:52:57,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731768777629, max timestamp=1731768777693Num uniques 1040; 2024-11-16T14:52:57,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=13280, suffix heap size=136128,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=290 2024-11-16T14:52:57,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=13280, suffix heap size=136128,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=290 2024-11-16T14:52:57,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731768777629, max timestamp=1731768777693Num uniques 1040; 2024-11-16T14:52:57,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=13280, suffix heap size=136128,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=290 2024-11-16T14:52:57,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:52:57,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:57,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:57,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:57,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:57,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:57,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:57,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731768777764, max timestamp=1731768777784Num uniques 1149; 2024-11-16T14:52:57,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-16T14:52:57,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731768777764, max timestamp=1731768777784Num uniques 1149; 2024-11-16T14:52:57,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-16T14:52:57,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731768777764, max timestamp=1731768777784Num uniques 1149; 2024-11-16T14:52:57,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-16T14:52:57,813 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/f330dfabbd924377be339e19c45a8f16 2024-11-16T14:52:57,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/6bb56529684a440a92e0342206c5fc7e is 50, key is test_row_0/B:col10/1731768776987/Put/seqid=0 2024-11-16T14:52:57,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742180_1356 (size=12151) 2024-11-16T14:52:57,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:52:57,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:57,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:57,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:57,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:57,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:57,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:52:57,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768837883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:57,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731768777879, max timestamp=1731768777894Num uniques 1158; 2024-11-16T14:52:57,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-11-16T14:52:57,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731768777879, max timestamp=1731768777894Num uniques 1158; 2024-11-16T14:52:57,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-11-16T14:52:57,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731768777879, max timestamp=1731768777894Num uniques 1158; 2024-11-16T14:52:57,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-11-16T14:52:57,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731768777969, max timestamp=1731768777983Num uniques 1017; 2024-11-16T14:52:57,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-16T14:52:57,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731768777969, max timestamp=1731768777983Num uniques 1017; 2024-11-16T14:52:57,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-16T14:52:57,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731768777969, max timestamp=1731768777983Num uniques 1017; 2024-11-16T14:52:57,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-16T14:52:58,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:52:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731768778009, max timestamp=1731768778034Num uniques 1076; 2024-11-16T14:52:58,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731768778009, max timestamp=1731768778034Num uniques 1076; 2024-11-16T14:52:58,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-16T14:52:58,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-16T14:52:58,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731768778009, max timestamp=1731768778034Num uniques 1076; 2024-11-16T14:52:58,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-16T14:52:58,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731768778072, max timestamp=1731768778088Num uniques 1135; 2024-11-16T14:52:58,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=10760, suffix heap size=146008,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=235 2024-11-16T14:52:58,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731768778072, max timestamp=1731768778088Num uniques 1135; 2024-11-16T14:52:58,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=10760, suffix heap size=146008,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=235 2024-11-16T14:52:58,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731768778072, max timestamp=1731768778088Num uniques 1135; 2024-11-16T14:52:58,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=10760, suffix heap size=146008,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=235 2024-11-16T14:52:58,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=394, cellSize=18044, totalHeapSize=41216, min timestamp=1731768778114, max timestamp=1731768778130Num uniques 1094; 2024-11-16T14:52:58,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=18044, suffix heap size=141744,new segment heap size=41216 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=394 2024-11-16T14:52:58,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=394, cellSize=18044, totalHeapSize=41216, min timestamp=1731768778114, max timestamp=1731768778130Num uniques 1094; 2024-11-16T14:52:58,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=394, cellSize=18044, totalHeapSize=41216, min timestamp=1731768778114, max timestamp=1731768778130Num uniques 1094; 2024-11-16T14:52:58,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=18044, suffix heap size=141744,new segment heap size=41216 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=394 2024-11-16T14:52:58,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=18044, suffix heap size=141744,new segment heap size=41216 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=394 2024-11-16T14:52:58,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:52:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:52:58,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768838168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768838169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768838170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768838171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768838171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/6bb56529684a440a92e0342206c5fc7e 2024-11-16T14:52:58,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/d67a3302ea5b437689c54afb2a778a67 is 50, key is test_row_0/C:col10/1731768776987/Put/seqid=0 2024-11-16T14:52:58,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742181_1357 (size=12151) 2024-11-16T14:52:58,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768838292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768838292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768838292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768838292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768838292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:52:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:52:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768838501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768838502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768838511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768838511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768838512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:52:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:52:58,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/d67a3302ea5b437689c54afb2a778a67 2024-11-16T14:52:58,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/f330dfabbd924377be339e19c45a8f16 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f330dfabbd924377be339e19c45a8f16 2024-11-16T14:52:58,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f330dfabbd924377be339e19c45a8f16, entries=200, sequenceid=183, filesize=38.8 K 2024-11-16T14:52:58,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/6bb56529684a440a92e0342206c5fc7e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6bb56529684a440a92e0342206c5fc7e 2024-11-16T14:52:58,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6bb56529684a440a92e0342206c5fc7e, entries=150, sequenceid=183, filesize=11.9 K 2024-11-16T14:52:58,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/d67a3302ea5b437689c54afb2a778a67 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/d67a3302ea5b437689c54afb2a778a67 2024-11-16T14:52:58,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/d67a3302ea5b437689c54afb2a778a67, entries=150, sequenceid=183, filesize=11.9 K 2024-11-16T14:52:58,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=167.72 KB/171750 for 49b096ff4ffb696a771be33a33a5e082 in 1656ms, sequenceid=183, compaction requested=true 2024-11-16T14:52:58,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:58,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:52:58,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:58,648 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:58,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:52:58,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:58,648 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:58,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:52:58,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:58,649 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:58,649 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/B is initiating minor compaction (all files) 2024-11-16T14:52:58,649 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/B in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,649 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6cb051435916468fbdbb25b1c896d4a2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/2798e09c8f7c43418e456dfe8aedb4b8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6bb56529684a440a92e0342206c5fc7e] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=33.4 K 2024-11-16T14:52:58,649 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:58,649 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/A is initiating minor compaction (all files) 2024-11-16T14:52:58,649 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/A in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,649 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a4563ebb52cb43a88484d7e4713747ff, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9bff21fdfe84642b43ac4320e31590a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f330dfabbd924377be339e19c45a8f16] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=91.3 K 2024-11-16T14:52:58,650 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,650 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a4563ebb52cb43a88484d7e4713747ff, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9bff21fdfe84642b43ac4320e31590a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f330dfabbd924377be339e19c45a8f16] 2024-11-16T14:52:58,650 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb051435916468fbdbb25b1c896d4a2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731768773684 2024-11-16T14:52:58,650 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4563ebb52cb43a88484d7e4713747ff, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731768773684 2024-11-16T14:52:58,650 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2798e09c8f7c43418e456dfe8aedb4b8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731768774905 2024-11-16T14:52:58,650 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9bff21fdfe84642b43ac4320e31590a, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731768774905 2024-11-16T14:52:58,651 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb56529684a440a92e0342206c5fc7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731768775757 2024-11-16T14:52:58,651 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f330dfabbd924377be339e19c45a8f16, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731768775757 2024-11-16T14:52:58,655 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:58,656 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:58,657 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/c35fa0741e8e49fc9785ac2cfb6b0ffb is 50, key is test_row_0/B:col10/1731768776987/Put/seqid=0 2024-11-16T14:52:58,663 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116df95e0af8b674c58a00a4bd5ba4c4345_49b096ff4ffb696a771be33a33a5e082 store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:58,665 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116df95e0af8b674c58a00a4bd5ba4c4345_49b096ff4ffb696a771be33a33a5e082, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:58,665 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116df95e0af8b674c58a00a4bd5ba4c4345_49b096ff4ffb696a771be33a33a5e082 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:58,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742182_1358 (size=12493) 2024-11-16T14:52:58,673 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/c35fa0741e8e49fc9785ac2cfb6b0ffb as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c35fa0741e8e49fc9785ac2cfb6b0ffb 2024-11-16T14:52:58,676 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/B of 49b096ff4ffb696a771be33a33a5e082 into c35fa0741e8e49fc9785ac2cfb6b0ffb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:58,676 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:58,676 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/B, priority=13, startTime=1731768778648; duration=0sec 2024-11-16T14:52:58,676 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:52:58,676 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:B 2024-11-16T14:52:58,676 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:52:58,678 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:52:58,678 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/C is initiating minor compaction (all files) 2024-11-16T14:52:58,678 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/C in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,678 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f2a7e22d505b40128a4ef36e12658f67, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/909d0c2815884013b6e2e3b7428e138e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/d67a3302ea5b437689c54afb2a778a67] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=33.4 K 2024-11-16T14:52:58,678 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a7e22d505b40128a4ef36e12658f67, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731768773684 2024-11-16T14:52:58,678 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 909d0c2815884013b6e2e3b7428e138e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731768774905 2024-11-16T14:52:58,679 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d67a3302ea5b437689c54afb2a778a67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731768775757 2024-11-16T14:52:58,688 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:58,688 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/02a96af20155429a802cd6620c1a0bd1 is 50, key is test_row_0/C:col10/1731768776987/Put/seqid=0 2024-11-16T14:52:58,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742183_1359 (size=4469) 2024-11-16T14:52:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:52:58,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742184_1360 (size=12493) 2024-11-16T14:52:58,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T14:52:58,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:52:58,778 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=167.72 KB heapSize=462.56 KB 2024-11-16T14:52:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:52:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:52:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:52:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:52:58,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111638e291620e924d66b68a00d9573695eb_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768778169/Put/seqid=0 2024-11-16T14:52:58,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742185_1361 (size=12454) 2024-11-16T14:52:58,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:52:58,805 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111638e291620e924d66b68a00d9573695eb_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111638e291620e924d66b68a00d9573695eb_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:58,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/0178370250634411bc9351edf05d842a, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:52:58,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/0178370250634411bc9351edf05d842a is 175, key is test_row_0/A:col10/1731768778169/Put/seqid=0 2024-11-16T14:52:58,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742186_1362 (size=31255) 2024-11-16T14:52:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:52:58,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:52:58,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768838830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768838830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768838831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768838831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768838838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768838940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768838950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768838950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,091 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:52:59,092 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/7887bbdfc6b84ef0ab6979e6578ec137 is 175, key is test_row_0/A:col10/1731768776987/Put/seqid=0 2024-11-16T14:52:59,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742187_1363 (size=31447) 2024-11-16T14:52:59,103 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/02a96af20155429a802cd6620c1a0bd1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/02a96af20155429a802cd6620c1a0bd1 2024-11-16T14:52:59,107 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/C of 49b096ff4ffb696a771be33a33a5e082 into 02a96af20155429a802cd6620c1a0bd1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:59,107 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:59,107 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/C, priority=13, startTime=1731768778648; duration=0sec 2024-11-16T14:52:59,107 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:59,107 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:C 2024-11-16T14:52:59,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768839151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768839161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768839161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,216 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=479, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/0178370250634411bc9351edf05d842a 2024-11-16T14:52:59,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/ca43bbc13c324c8d881542b7e2bb34e4 is 50, key is test_row_0/B:col10/1731768778169/Put/seqid=0 2024-11-16T14:52:59,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742188_1364 (size=12301) 2024-11-16T14:52:59,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768839350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768839350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768839470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768839481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768839481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,500 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/7887bbdfc6b84ef0ab6979e6578ec137 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/7887bbdfc6b84ef0ab6979e6578ec137 2024-11-16T14:52:59,503 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/A of 49b096ff4ffb696a771be33a33a5e082 into 7887bbdfc6b84ef0ab6979e6578ec137(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:52:59,503 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:52:59,503 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/A, priority=13, startTime=1731768778648; duration=0sec 2024-11-16T14:52:59,503 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:52:59,503 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:A 2024-11-16T14:52:59,633 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/ca43bbc13c324c8d881542b7e2bb34e4 2024-11-16T14:52:59,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/60b6f50175f54a50bc99928736ddb776 is 50, key is test_row_0/C:col10/1731768778169/Put/seqid=0 2024-11-16T14:52:59,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742189_1365 (size=12301) 2024-11-16T14:52:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:52:59,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768839991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768839991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:52:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:52:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768839991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:52:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:52:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:00,044 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/60b6f50175f54a50bc99928736ddb776 2024-11-16T14:53:00,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/0178370250634411bc9351edf05d842a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/0178370250634411bc9351edf05d842a 2024-11-16T14:53:00,050 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/0178370250634411bc9351edf05d842a, entries=150, sequenceid=479, filesize=30.5 K 2024-11-16T14:53:00,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/ca43bbc13c324c8d881542b7e2bb34e4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca43bbc13c324c8d881542b7e2bb34e4 2024-11-16T14:53:00,056 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca43bbc13c324c8d881542b7e2bb34e4, entries=150, sequenceid=479, filesize=12.0 K 2024-11-16T14:53:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/60b6f50175f54a50bc99928736ddb776 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/60b6f50175f54a50bc99928736ddb776 2024-11-16T14:53:00,063 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/60b6f50175f54a50bc99928736ddb776, entries=150, sequenceid=479, filesize=12.0 K 2024-11-16T14:53:00,064 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~462.52 KB/473616, currentSize=20.13 KB/20610 for 49b096ff4ffb696a771be33a33a5e082 in 1287ms, sequenceid=479, compaction requested=false 2024-11-16T14:53:00,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:00,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:00,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-16T14:53:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-16T14:53:00,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-16T14:53:00,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5110 sec 2024-11-16T14:53:00,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.5220 sec 2024-11-16T14:53:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:00,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:53:00,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:53:00,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:00,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:53:00,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:00,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:53:00,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:00,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eb2dc75287894b04a5cd7a6feaa9d5f2_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768780374/Put/seqid=0 2024-11-16T14:53:00,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742190_1366 (size=20074) 2024-11-16T14:53:00,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768780433, max timestamp=1731768780456Num uniques 859; 2024-11-16T14:53:00,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768780433, max timestamp=1731768780456Num uniques 859; 2024-11-16T14:53:00,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:53:00,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:00,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768780433, max timestamp=1731768780456Num uniques 859; 2024-11-16T14:53:00,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:00,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768840530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:00,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:00,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768840542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:00,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768840650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768840650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:00,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:00,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:00,799 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eb2dc75287894b04a5cd7a6feaa9d5f2_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eb2dc75287894b04a5cd7a6feaa9d5f2_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:00,800 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/f423fa6df0dd4516810ebc440078fc89, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:00,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/f423fa6df0dd4516810ebc440078fc89 is 175, key is test_row_0/A:col10/1731768780374/Put/seqid=0 2024-11-16T14:53:00,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742191_1367 (size=57333) 2024-11-16T14:53:00,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768840861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:00,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768840871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:01,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768841010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:01,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768841010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:01,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768841010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:01,019 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in 49b096ff4ffb696a771be33a33a5e082 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:01,019 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in 49b096ff4ffb696a771be33a33a5e082 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:01,019 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in 49b096ff4ffb696a771be33a33a5e082 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:01,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768841180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:01,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768841191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:01,209 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=491, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/f423fa6df0dd4516810ebc440078fc89 2024-11-16T14:53:01,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/20af16ba8bd6477b98ea2e7998d00cbc is 50, key is test_row_0/B:col10/1731768780374/Put/seqid=0 2024-11-16T14:53:01,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742192_1368 (size=12301) 2024-11-16T14:53:01,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/20af16ba8bd6477b98ea2e7998d00cbc 2024-11-16T14:53:01,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/bfedfc03376a4d3abac6c973cc78cdf6 is 50, key is test_row_0/C:col10/1731768780374/Put/seqid=0 2024-11-16T14:53:01,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742193_1369 (size=12301) 2024-11-16T14:53:01,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/bfedfc03376a4d3abac6c973cc78cdf6 2024-11-16T14:53:01,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/f423fa6df0dd4516810ebc440078fc89 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f423fa6df0dd4516810ebc440078fc89 2024-11-16T14:53:01,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f423fa6df0dd4516810ebc440078fc89, entries=300, sequenceid=491, filesize=56.0 K 2024-11-16T14:53:01,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/20af16ba8bd6477b98ea2e7998d00cbc as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/20af16ba8bd6477b98ea2e7998d00cbc 2024-11-16T14:53:01,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/20af16ba8bd6477b98ea2e7998d00cbc, entries=150, sequenceid=491, filesize=12.0 K 2024-11-16T14:53:01,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/bfedfc03376a4d3abac6c973cc78cdf6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/bfedfc03376a4d3abac6c973cc78cdf6 2024-11-16T14:53:01,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/bfedfc03376a4d3abac6c973cc78cdf6, entries=150, sequenceid=491, filesize=12.0 K 2024-11-16T14:53:01,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 49b096ff4ffb696a771be33a33a5e082 in 1286ms, sequenceid=491, compaction requested=true 2024-11-16T14:53:01,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:01,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:01,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:01,663 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:01,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:01,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:01,663 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:01,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:01,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:01,664 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120035 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:01,664 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:01,664 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/A is initiating minor compaction (all files) 2024-11-16T14:53:01,664 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/B is initiating minor compaction (all files) 2024-11-16T14:53:01,664 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/B in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:01,664 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/A in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:01,664 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/7887bbdfc6b84ef0ab6979e6578ec137, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/0178370250634411bc9351edf05d842a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f423fa6df0dd4516810ebc440078fc89] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=117.2 K 2024-11-16T14:53:01,664 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c35fa0741e8e49fc9785ac2cfb6b0ffb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca43bbc13c324c8d881542b7e2bb34e4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/20af16ba8bd6477b98ea2e7998d00cbc] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=36.2 K 2024-11-16T14:53:01,664 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:01,664 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/7887bbdfc6b84ef0ab6979e6578ec137, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/0178370250634411bc9351edf05d842a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f423fa6df0dd4516810ebc440078fc89] 2024-11-16T14:53:01,664 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting c35fa0741e8e49fc9785ac2cfb6b0ffb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731768775757 2024-11-16T14:53:01,664 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7887bbdfc6b84ef0ab6979e6578ec137, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731768775757 2024-11-16T14:53:01,664 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ca43bbc13c324c8d881542b7e2bb34e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731768778169 2024-11-16T14:53:01,665 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0178370250634411bc9351edf05d842a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731768778169 2024-11-16T14:53:01,665 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 20af16ba8bd6477b98ea2e7998d00cbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731768778837 2024-11-16T14:53:01,667 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f423fa6df0dd4516810ebc440078fc89, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731768778830 2024-11-16T14:53:01,674 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:01,674 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/ca6e44bc431f415481ec9888302fdb76 is 50, key is test_row_0/B:col10/1731768780374/Put/seqid=0 2024-11-16T14:53:01,683 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:01,691 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111627e6fe8bceb949219a4a7f49b73fef4a_49b096ff4ffb696a771be33a33a5e082 store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:01,693 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111627e6fe8bceb949219a4a7f49b73fef4a_49b096ff4ffb696a771be33a33a5e082, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:01,694 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111627e6fe8bceb949219a4a7f49b73fef4a_49b096ff4ffb696a771be33a33a5e082 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:01,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742194_1370 (size=12745) 2024-11-16T14:53:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:01,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731768780522, max timestamp=1731768781700Num uniques 1068; 2024-11-16T14:53:01,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T14:53:01,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731768780522, max timestamp=1731768781700Num uniques 1018; 2024-11-16T14:53:01,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731768780522, max timestamp=1731768781700Num uniques 1018; 2024-11-16T14:53:01,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T14:53:01,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-16T14:53:01,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-11-16T14:53:01,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:53:01,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:01,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:53:01,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:01,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:53:01,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:01,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742195_1371 (size=4469) 2024-11-16T14:53:01,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a1fe4123617e4fdba08e907c810b3c41_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768780522/Put/seqid=0 2024-11-16T14:53:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T14:53:01,721 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:01,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-16T14:53:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:53:01,733 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:01,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742196_1372 (size=14994) 2024-11-16T14:53:01,734 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:01,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:01,735 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:01,740 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a1fe4123617e4fdba08e907c810b3c41_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a1fe4123617e4fdba08e907c810b3c41_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:01,743 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/cefdbfad5f384b999ba4d0db003ff5c1, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:01,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/cefdbfad5f384b999ba4d0db003ff5c1 is 175, key is test_row_0/A:col10/1731768780522/Put/seqid=0 2024-11-16T14:53:01,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742197_1373 (size=39949) 2024-11-16T14:53:01,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768781792, max timestamp=1731768781816Num uniques 859; 2024-11-16T14:53:01,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768781792, max timestamp=1731768781816Num uniques 859; 2024-11-16T14:53:01,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768781792, max timestamp=1731768781816Num uniques 859; 2024-11-16T14:53:01,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:53:01,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:53:01,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:53:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:53:01,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:01,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:01,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:01,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:01,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:01,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:01,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731768781855, max timestamp=1731768781911Num uniques 1068; 2024-11-16T14:53:01,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731768781855, max timestamp=1731768781911Num uniques 1068; 2024-11-16T14:53:01,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-16T14:53:01,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731768781855, max timestamp=1731768781911Num uniques 1068; 2024-11-16T14:53:01,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-16T14:53:01,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-16T14:53:02,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731768781986, max timestamp=1731768782020Num uniques 1077; 2024-11-16T14:53:02,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-16T14:53:02,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731768781986, max timestamp=1731768782020Num uniques 1077; 2024-11-16T14:53:02,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731768781986, max timestamp=1731768782020Num uniques 1077; 2024-11-16T14:53:02,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-16T14:53:02,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-16T14:53:02,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:02,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:02,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:53:02,099 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/ca6e44bc431f415481ec9888302fdb76 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca6e44bc431f415481ec9888302fdb76 2024-11-16T14:53:02,104 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/B of 49b096ff4ffb696a771be33a33a5e082 into ca6e44bc431f415481ec9888302fdb76(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:02,104 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:02,105 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/B, priority=13, startTime=1731768781663; duration=0sec 2024-11-16T14:53:02,105 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:02,105 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:B 2024-11-16T14:53:02,105 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:02,107 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:02,107 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/C is initiating minor compaction (all files) 2024-11-16T14:53:02,107 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/C in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,107 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#A#compaction#313 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:02,107 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/02a96af20155429a802cd6620c1a0bd1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/60b6f50175f54a50bc99928736ddb776, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/bfedfc03376a4d3abac6c973cc78cdf6] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=36.2 K 2024-11-16T14:53:02,108 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 02a96af20155429a802cd6620c1a0bd1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731768775757 2024-11-16T14:53:02,108 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/45ef200c36844bebb4084886e7347510 is 175, key is test_row_0/A:col10/1731768780374/Put/seqid=0 2024-11-16T14:53:02,109 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b6f50175f54a50bc99928736ddb776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731768778169 2024-11-16T14:53:02,110 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting bfedfc03376a4d3abac6c973cc78cdf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731768778837 2024-11-16T14:53:02,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742198_1374 (size=31699) 2024-11-16T14:53:02,117 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:02,117 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/45ef200c36844bebb4084886e7347510 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/45ef200c36844bebb4084886e7347510 2024-11-16T14:53:02,117 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/9b2c94192501402aad7f04118beb53ac is 50, key is test_row_0/C:col10/1731768780374/Put/seqid=0 2024-11-16T14:53:02,120 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/A of 49b096ff4ffb696a771be33a33a5e082 into 45ef200c36844bebb4084886e7347510(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:02,120 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:02,120 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/A, priority=13, startTime=1731768781663; duration=0sec 2024-11-16T14:53:02,120 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:02,120 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:A 2024-11-16T14:53:02,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742199_1375 (size=12745) 2024-11-16T14:53:02,135 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/9b2c94192501402aad7f04118beb53ac as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/9b2c94192501402aad7f04118beb53ac 2024-11-16T14:53:02,150 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=529, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/cefdbfad5f384b999ba4d0db003ff5c1 2024-11-16T14:53:02,154 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/C of 49b096ff4ffb696a771be33a33a5e082 into 9b2c94192501402aad7f04118beb53ac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:02,154 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:02,154 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/C, priority=13, startTime=1731768781663; duration=0sec 2024-11-16T14:53:02,154 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:02,154 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:C 2024-11-16T14:53:02,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/f11b002ce34d4cb2b54c5fbbdff99785 is 50, key is test_row_0/B:col10/1731768780522/Put/seqid=0 2024-11-16T14:53:02,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731768782123, max timestamp=1731768782163Num uniques 1086; 2024-11-16T14:53:02,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-16T14:53:02,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731768782123, max timestamp=1731768782163Num uniques 1086; 2024-11-16T14:53:02,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-16T14:53:02,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731768782123, max timestamp=1731768782163Num uniques 1086; 2024-11-16T14:53:02,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-16T14:53:02,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742200_1376 (size=12301) 2024-11-16T14:53:02,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:02,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731768782236, max timestamp=1731768782266Num uniques 1045; 2024-11-16T14:53:02,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731768782236, max timestamp=1731768782266Num uniques 1045; 2024-11-16T14:53:02,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-16T14:53:02,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-16T14:53:02,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731768782236, max timestamp=1731768782266Num uniques 1045; 2024-11-16T14:53:02,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-16T14:53:02,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731768782331, max timestamp=1731768782357Num uniques 1154; 2024-11-16T14:53:02,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-16T14:53:02,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731768782331, max timestamp=1731768782357Num uniques 1154; 2024-11-16T14:53:02,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-16T14:53:02,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731768782331, max timestamp=1731768782357Num uniques 1154; 2024-11-16T14:53:02,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-16T14:53:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:53:02,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731768782428, max timestamp=1731768782446Num uniques 1063; 2024-11-16T14:53:02,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-16T14:53:02,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731768782428, max timestamp=1731768782446Num uniques 1063; 2024-11-16T14:53:02,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-16T14:53:02,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731768782428, max timestamp=1731768782446Num uniques 1063; 2024-11-16T14:53:02,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-16T14:53:02,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:02,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:02,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731768782493, max timestamp=1731768782526Num uniques 1072; 2024-11-16T14:53:02,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-16T14:53:02,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731768782493, max timestamp=1731768782526Num uniques 1072; 2024-11-16T14:53:02,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731768782493, max timestamp=1731768782526Num uniques 1072; 2024-11-16T14:53:02,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-16T14:53:02,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-16T14:53:02,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/f11b002ce34d4cb2b54c5fbbdff99785 2024-11-16T14:53:02,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/a9415e32074c47c1ba1a303446034434 is 50, key is test_row_0/C:col10/1731768780522/Put/seqid=0 2024-11-16T14:53:02,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742201_1377 (size=12301) 2024-11-16T14:53:02,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731768782574, max timestamp=1731768782601Num uniques 1031; 2024-11-16T14:53:02,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731768782574, max timestamp=1731768782601Num uniques 1031; 2024-11-16T14:53:02,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-16T14:53:02,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-16T14:53:02,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731768782574, max timestamp=1731768782601Num uniques 1031; 2024-11-16T14:53:02,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-16T14:53:02,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:02,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:02,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731768782674, max timestamp=1731768782704Num uniques 1090; 2024-11-16T14:53:02,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731768782674, max timestamp=1731768782704Num uniques 1090; 2024-11-16T14:53:02,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-16T14:53:02,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-16T14:53:02,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731768782674, max timestamp=1731768782704Num uniques 1090; 2024-11-16T14:53:02,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-16T14:53:02,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731768782730, max timestamp=1731768782798Num uniques 1049; 2024-11-16T14:53:02,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-16T14:53:02,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731768782730, max timestamp=1731768782798Num uniques 1049; 2024-11-16T14:53:02,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731768782730, max timestamp=1731768782798Num uniques 1049; 2024-11-16T14:53:02,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-16T14:53:02,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-16T14:53:02,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:53:02,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731768782867, max timestamp=1731768782886Num uniques 1108; 2024-11-16T14:53:02,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731768782867, max timestamp=1731768782886Num uniques 1108; 2024-11-16T14:53:02,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-16T14:53:02,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-16T14:53:02,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731768782867, max timestamp=1731768782886Num uniques 1108; 2024-11-16T14:53:02,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-16T14:53:02,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:02,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:02,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:02,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:02,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731768782922, max timestamp=1731768782957Num uniques 1017; 2024-11-16T14:53:02,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-16T14:53:02,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731768782922, max timestamp=1731768782957Num uniques 1017; 2024-11-16T14:53:02,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-16T14:53:02,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731768782922, max timestamp=1731768782957Num uniques 1017; 2024-11-16T14:53:02,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-16T14:53:02,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/a9415e32074c47c1ba1a303446034434 2024-11-16T14:53:02,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/cefdbfad5f384b999ba4d0db003ff5c1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/cefdbfad5f384b999ba4d0db003ff5c1 2024-11-16T14:53:02,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/cefdbfad5f384b999ba4d0db003ff5c1, entries=200, sequenceid=529, filesize=39.0 K 2024-11-16T14:53:02,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/f11b002ce34d4cb2b54c5fbbdff99785 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f11b002ce34d4cb2b54c5fbbdff99785 2024-11-16T14:53:02,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f11b002ce34d4cb2b54c5fbbdff99785, entries=150, sequenceid=529, filesize=12.0 K 2024-11-16T14:53:02,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/a9415e32074c47c1ba1a303446034434 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a9415e32074c47c1ba1a303446034434 2024-11-16T14:53:03,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a9415e32074c47c1ba1a303446034434, entries=150, sequenceid=529, filesize=12.0 K 2024-11-16T14:53:03,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=107.34 KB/109920 for 49b096ff4ffb696a771be33a33a5e082 in 1298ms, sequenceid=529, compaction requested=false 2024-11-16T14:53:03,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:03,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=114.05 KB heapSize=315.09 KB 2024-11-16T14:53:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:53:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:53:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:53:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:03,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116830502e866804162b8bcf370bff88689_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768782991/Put/seqid=0 2024-11-16T14:53:03,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742202_1378 (size=20074) 2024-11-16T14:53:03,020 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:03,023 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116830502e866804162b8bcf370bff88689_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116830502e866804162b8bcf370bff88689_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:03,024 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/299da3fe48c848d59c3412b1906d76f4, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:03,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/299da3fe48c848d59c3412b1906d76f4 is 175, key is test_row_0/A:col10/1731768782991/Put/seqid=0 2024-11-16T14:53:03,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742203_1379 (size=57333) 2024-11-16T14:53:03,027 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=770, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/299da3fe48c848d59c3412b1906d76f4 2024-11-16T14:53:03,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/543c249f9a25434ebef8af3399e8eba7 is 50, key is test_row_0/B:col10/1731768782991/Put/seqid=0 2024-11-16T14:53:03,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742204_1380 (size=12301) 2024-11-16T14:53:03,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768843038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768843039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768843040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768843048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768843048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,051 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in 49b096ff4ffb696a771be33a33a5e082 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:03,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:03,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768843160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768843160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768843160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768843160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:03,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:03,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768843371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768843371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768843371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768843371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:03,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:03,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=770 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/543c249f9a25434ebef8af3399e8eba7 2024-11-16T14:53:03,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/4cdbca37413540d08850a616d692e901 is 50, key is test_row_0/C:col10/1731768782991/Put/seqid=0 2024-11-16T14:53:03,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742205_1381 (size=12301) 2024-11-16T14:53:03,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:03,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768843680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768843681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768843691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768843691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:03,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:03,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:03,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=770 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/4cdbca37413540d08850a616d692e901 2024-11-16T14:53:03,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/299da3fe48c848d59c3412b1906d76f4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/299da3fe48c848d59c3412b1906d76f4 2024-11-16T14:53:03,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/299da3fe48c848d59c3412b1906d76f4, entries=300, sequenceid=770, filesize=56.0 K 2024-11-16T14:53:03,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/543c249f9a25434ebef8af3399e8eba7 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/543c249f9a25434ebef8af3399e8eba7 2024-11-16T14:53:03,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/543c249f9a25434ebef8af3399e8eba7, entries=150, sequenceid=770, filesize=12.0 K 2024-11-16T14:53:03,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/4cdbca37413540d08850a616d692e901 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4cdbca37413540d08850a616d692e901 2024-11-16T14:53:03,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:03,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:03,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:03,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4cdbca37413540d08850a616d692e901, entries=150, sequenceid=770, filesize=12.0 K 2024-11-16T14:53:03,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~315.05 KB/322608, currentSize=67.09 KB/68700 for 49b096ff4ffb696a771be33a33a5e082 in 874ms, sequenceid=770, compaction requested=true 2024-11-16T14:53:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:03,878 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T14:53:03,879 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:53:03,882 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:03,882 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/A is initiating minor compaction (all files) 2024-11-16T14:53:03,883 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/A in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,883 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/45ef200c36844bebb4084886e7347510, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/cefdbfad5f384b999ba4d0db003ff5c1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/299da3fe48c848d59c3412b1906d76f4] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=126.0 K 2024-11-16T14:53:03,883 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,883 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/45ef200c36844bebb4084886e7347510, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/cefdbfad5f384b999ba4d0db003ff5c1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/299da3fe48c848d59c3412b1906d76f4] 2024-11-16T14:53:03,883 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:03,883 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/B is initiating minor compaction (all files) 2024-11-16T14:53:03,883 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/B in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:03,883 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca6e44bc431f415481ec9888302fdb76, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f11b002ce34d4cb2b54c5fbbdff99785, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/543c249f9a25434ebef8af3399e8eba7] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=36.5 K 2024-11-16T14:53:03,883 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 45ef200c36844bebb4084886e7347510, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731768778837 2024-11-16T14:53:03,885 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca6e44bc431f415481ec9888302fdb76, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731768778837 2024-11-16T14:53:03,885 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting cefdbfad5f384b999ba4d0db003ff5c1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1731768780522 2024-11-16T14:53:03,885 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f11b002ce34d4cb2b54c5fbbdff99785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1731768780522 2024-11-16T14:53:03,885 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 299da3fe48c848d59c3412b1906d76f4, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=770, earliestPutTs=1731768782951 2024-11-16T14:53:03,885 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 543c249f9a25434ebef8af3399e8eba7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=770, earliestPutTs=1731768782951 2024-11-16T14:53:03,892 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:03,893 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:03,894 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/5b50aef6f61f4799b39949231393a9f0 is 50, key is test_row_0/B:col10/1731768782991/Put/seqid=0 2024-11-16T14:53:03,895 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116735923b0537f40eb9803ab5c3f6127d0_49b096ff4ffb696a771be33a33a5e082 store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:03,897 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116735923b0537f40eb9803ab5c3f6127d0_49b096ff4ffb696a771be33a33a5e082, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:03,897 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116735923b0537f40eb9803ab5c3f6127d0_49b096ff4ffb696a771be33a33a5e082 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:03,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742206_1382 (size=12847) 2024-11-16T14:53:03,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742207_1383 (size=4469) 2024-11-16T14:53:04,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T14:53:04,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:04,031 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:53:04,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:53:04,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:04,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:53:04,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:04,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:53:04,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:04,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162b985921a22f4ce7851ff9b52ee2fb50_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768783035/Put/seqid=0 2024-11-16T14:53:04,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742208_1384 (size=12454) 2024-11-16T14:53:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:04,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:04,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768844252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768844252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768844266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768844267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,318 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#A#compaction#322 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:04,318 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/88c0cba22d5c40b6aadb0547da2159f6 is 175, key is test_row_0/A:col10/1731768782991/Put/seqid=0 2024-11-16T14:53:04,321 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/5b50aef6f61f4799b39949231393a9f0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5b50aef6f61f4799b39949231393a9f0 2024-11-16T14:53:04,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742209_1385 (size=31801) 2024-11-16T14:53:04,325 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/B of 49b096ff4ffb696a771be33a33a5e082 into 5b50aef6f61f4799b39949231393a9f0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:04,325 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:04,325 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/B, priority=13, startTime=1731768783878; duration=0sec 2024-11-16T14:53:04,325 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:04,325 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:B 2024-11-16T14:53:04,325 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:04,326 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:04,326 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/C is initiating minor compaction (all files) 2024-11-16T14:53:04,327 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/C in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:04,327 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/9b2c94192501402aad7f04118beb53ac, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a9415e32074c47c1ba1a303446034434, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4cdbca37413540d08850a616d692e901] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=36.5 K 2024-11-16T14:53:04,327 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b2c94192501402aad7f04118beb53ac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731768778837 2024-11-16T14:53:04,327 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9415e32074c47c1ba1a303446034434, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1731768780522 2024-11-16T14:53:04,327 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cdbca37413540d08850a616d692e901, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=770, earliestPutTs=1731768782951 2024-11-16T14:53:04,333 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:04,333 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/4e2a3d1a0e284362b17ea14d83aca73b is 50, key is test_row_0/C:col10/1731768782991/Put/seqid=0 2024-11-16T14:53:04,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742210_1386 (size=12847) 2024-11-16T14:53:04,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768844381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768844381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768844381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768844390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:04,458 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162b985921a22f4ce7851ff9b52ee2fb50_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162b985921a22f4ce7851ff9b52ee2fb50_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:04,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/06a3f00d5934493aaef2aaee8d687595, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:04,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/06a3f00d5934493aaef2aaee8d687595 is 175, key is test_row_0/A:col10/1731768783035/Put/seqid=0 2024-11-16T14:53:04,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742211_1387 (size=31255) 2024-11-16T14:53:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768844601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768844601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768844601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768844611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,742 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/88c0cba22d5c40b6aadb0547da2159f6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/88c0cba22d5c40b6aadb0547da2159f6 2024-11-16T14:53:04,744 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/4e2a3d1a0e284362b17ea14d83aca73b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4e2a3d1a0e284362b17ea14d83aca73b 2024-11-16T14:53:04,760 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/A of 49b096ff4ffb696a771be33a33a5e082 into 88c0cba22d5c40b6aadb0547da2159f6(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:04,760 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:04,760 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/A, priority=13, startTime=1731768783878; duration=0sec 2024-11-16T14:53:04,760 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:04,760 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:A 2024-11-16T14:53:04,765 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/C of 49b096ff4ffb696a771be33a33a5e082 into 4e2a3d1a0e284362b17ea14d83aca73b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:04,765 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:04,765 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/C, priority=13, startTime=1731768783878; duration=0sec 2024-11-16T14:53:04,765 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:04,765 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:C 2024-11-16T14:53:04,865 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=783, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/06a3f00d5934493aaef2aaee8d687595 2024-11-16T14:53:04,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/42d1325b30d64c6cb710c4df4c11e7a9 is 50, key is test_row_0/B:col10/1731768783035/Put/seqid=0 2024-11-16T14:53:04,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742212_1388 (size=12301) 2024-11-16T14:53:04,876 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/42d1325b30d64c6cb710c4df4c11e7a9 2024-11-16T14:53:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/1af28fc863d242688a51d44c1dff1ad5 is 50, key is test_row_0/C:col10/1731768783035/Put/seqid=0 2024-11-16T14:53:04,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742213_1389 (size=12301) 2024-11-16T14:53:04,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768844920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768844921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768844921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:04,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768844933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/1af28fc863d242688a51d44c1dff1ad5 2024-11-16T14:53:05,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/06a3f00d5934493aaef2aaee8d687595 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/06a3f00d5934493aaef2aaee8d687595 2024-11-16T14:53:05,297 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/06a3f00d5934493aaef2aaee8d687595, entries=150, sequenceid=783, filesize=30.5 K 2024-11-16T14:53:05,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/42d1325b30d64c6cb710c4df4c11e7a9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/42d1325b30d64c6cb710c4df4c11e7a9 2024-11-16T14:53:05,302 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/42d1325b30d64c6cb710c4df4c11e7a9, entries=150, sequenceid=783, filesize=12.0 K 2024-11-16T14:53:05,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/1af28fc863d242688a51d44c1dff1ad5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/1af28fc863d242688a51d44c1dff1ad5 2024-11-16T14:53:05,309 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/1af28fc863d242688a51d44c1dff1ad5, entries=150, sequenceid=783, filesize=12.0 K 2024-11-16T14:53:05,311 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 49b096ff4ffb696a771be33a33a5e082 in 1279ms, sequenceid=783, compaction requested=false 2024-11-16T14:53:05,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:05,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:05,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-16T14:53:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-16T14:53:05,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-16T14:53:05,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5780 sec 2024-11-16T14:53:05,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.5940 sec 2024-11-16T14:53:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:05,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:53:05,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:53:05,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:05,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:53:05,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:05,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:53:05,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:05,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116542fcac8ed9f43cf8273be8bfff98d15_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768784250/Put/seqid=0 2024-11-16T14:53:05,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742214_1390 (size=14994) 2024-11-16T14:53:05,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768845460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768845462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768845470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768845471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768845580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768845580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768845580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768845590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768845790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768845802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768845802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768845811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:05,856 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:05,868 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116542fcac8ed9f43cf8273be8bfff98d15_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116542fcac8ed9f43cf8273be8bfff98d15_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:05,869 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/4c3b611beacd4022b208af9921e47e5b, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:05,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/4c3b611beacd4022b208af9921e47e5b is 175, key is test_row_0/A:col10/1731768784250/Put/seqid=0 2024-11-16T14:53:05,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742215_1391 (size=39949) 2024-11-16T14:53:05,880 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=806, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/4c3b611beacd4022b208af9921e47e5b 2024-11-16T14:53:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T14:53:05,893 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:05,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-16T14:53:05,897 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:53:05,899 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:05,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:05,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/cfc00911482d427ab1eb221832cece5f is 50, key is test_row_0/B:col10/1731768784250/Put/seqid=0 2024-11-16T14:53:05,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742216_1392 (size=12301) 2024-11-16T14:53:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:53:06,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T14:53:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:06,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:06,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:06,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768846111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768846111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768846111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768846130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T14:53:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:06,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:53:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:06,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=806 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/cfc00911482d427ab1eb221832cece5f 2024-11-16T14:53:06,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/e5cc0d76ff284191935a09c3db830dd3 is 50, key is test_row_0/C:col10/1731768784250/Put/seqid=0 2024-11-16T14:53:06,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742217_1393 (size=12301) 2024-11-16T14:53:06,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=806 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/e5cc0d76ff284191935a09c3db830dd3 2024-11-16T14:53:06,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/4c3b611beacd4022b208af9921e47e5b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/4c3b611beacd4022b208af9921e47e5b 2024-11-16T14:53:06,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/4c3b611beacd4022b208af9921e47e5b, entries=200, sequenceid=806, filesize=39.0 K 2024-11-16T14:53:06,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/cfc00911482d427ab1eb221832cece5f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/cfc00911482d427ab1eb221832cece5f 2024-11-16T14:53:06,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/cfc00911482d427ab1eb221832cece5f, entries=150, sequenceid=806, filesize=12.0 K 2024-11-16T14:53:06,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/e5cc0d76ff284191935a09c3db830dd3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/e5cc0d76ff284191935a09c3db830dd3 2024-11-16T14:53:06,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/e5cc0d76ff284191935a09c3db830dd3, entries=150, sequenceid=806, filesize=12.0 K 2024-11-16T14:53:06,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 49b096ff4ffb696a771be33a33a5e082 in 900ms, sequenceid=806, compaction requested=true 2024-11-16T14:53:06,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:06,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:06,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:06,345 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:06,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:06,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:06,345 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:06,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:06,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:06,345 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:06,345 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:06,345 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/A is initiating minor compaction (all files) 2024-11-16T14:53:06,345 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/B is initiating minor compaction (all files) 2024-11-16T14:53:06,345 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/A in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:06,345 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/B in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:06,346 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5b50aef6f61f4799b39949231393a9f0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/42d1325b30d64c6cb710c4df4c11e7a9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/cfc00911482d427ab1eb221832cece5f] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=36.6 K 2024-11-16T14:53:06,346 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/88c0cba22d5c40b6aadb0547da2159f6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/06a3f00d5934493aaef2aaee8d687595, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/4c3b611beacd4022b208af9921e47e5b] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=100.6 K 2024-11-16T14:53:06,346 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:06,346 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/88c0cba22d5c40b6aadb0547da2159f6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/06a3f00d5934493aaef2aaee8d687595, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/4c3b611beacd4022b208af9921e47e5b] 2024-11-16T14:53:06,346 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b50aef6f61f4799b39949231393a9f0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=770, earliestPutTs=1731768782951 2024-11-16T14:53:06,346 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88c0cba22d5c40b6aadb0547da2159f6, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=770, earliestPutTs=1731768782951 2024-11-16T14:53:06,346 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 42d1325b30d64c6cb710c4df4c11e7a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731768783035 2024-11-16T14:53:06,346 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a3f00d5934493aaef2aaee8d687595, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731768783035 2024-11-16T14:53:06,346 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting cfc00911482d427ab1eb221832cece5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=806, earliestPutTs=1731768784250 2024-11-16T14:53:06,346 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c3b611beacd4022b208af9921e47e5b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=806, earliestPutTs=1731768784250 2024-11-16T14:53:06,352 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:06,354 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:06,355 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/fe012abc0c984c37a683d79ab23b606e is 50, key is test_row_0/B:col10/1731768784250/Put/seqid=0 2024-11-16T14:53:06,355 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411167aacd49f97f04f969033e3a88a770b69_49b096ff4ffb696a771be33a33a5e082 store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:06,357 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411167aacd49f97f04f969033e3a88a770b69_49b096ff4ffb696a771be33a33a5e082, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:06,357 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167aacd49f97f04f969033e3a88a770b69_49b096ff4ffb696a771be33a33a5e082 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:06,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742218_1394 (size=12949) 2024-11-16T14:53:06,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T14:53:06,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:06,363 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:53:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:53:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:53:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:53:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:06,368 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/fe012abc0c984c37a683d79ab23b606e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/fe012abc0c984c37a683d79ab23b606e 2024-11-16T14:53:06,372 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/B of 49b096ff4ffb696a771be33a33a5e082 into fe012abc0c984c37a683d79ab23b606e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:06,372 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:06,372 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/B, priority=13, startTime=1731768786345; duration=0sec 2024-11-16T14:53:06,372 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:06,372 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:B 2024-11-16T14:53:06,372 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:06,373 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:06,373 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/C is initiating minor compaction (all files) 2024-11-16T14:53:06,373 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/C in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:06,373 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4e2a3d1a0e284362b17ea14d83aca73b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/1af28fc863d242688a51d44c1dff1ad5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/e5cc0d76ff284191935a09c3db830dd3] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=36.6 K 2024-11-16T14:53:06,373 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e2a3d1a0e284362b17ea14d83aca73b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=770, earliestPutTs=1731768782951 2024-11-16T14:53:06,373 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 1af28fc863d242688a51d44c1dff1ad5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731768783035 2024-11-16T14:53:06,374 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting e5cc0d76ff284191935a09c3db830dd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=806, earliestPutTs=1731768784250 2024-11-16T14:53:06,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742219_1395 (size=4469) 2024-11-16T14:53:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bd5473650a054171bff8c3251c0e2f56_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768785460/Put/seqid=0 2024-11-16T14:53:06,382 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#A#compaction#331 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:06,382 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/53cd520af3934dbc9e27a3f7f4a0c21c is 175, key is test_row_0/A:col10/1731768784250/Put/seqid=0 2024-11-16T14:53:06,398 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:06,399 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/b03a1b41dae446129cb5010e932b7cff is 50, key is test_row_0/C:col10/1731768784250/Put/seqid=0 2024-11-16T14:53:06,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742221_1397 (size=31903) 2024-11-16T14:53:06,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742220_1396 (size=12454) 2024-11-16T14:53:06,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742222_1398 (size=12949) 2024-11-16T14:53:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:53:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:06,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768846666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768846675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768846675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768846676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768846780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768846790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768846790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768846791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:06,809 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/53cd520af3934dbc9e27a3f7f4a0c21c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/53cd520af3934dbc9e27a3f7f4a0c21c 2024-11-16T14:53:06,813 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/A of 49b096ff4ffb696a771be33a33a5e082 into 53cd520af3934dbc9e27a3f7f4a0c21c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:06,813 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:06,813 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/A, priority=13, startTime=1731768786344; duration=0sec 2024-11-16T14:53:06,813 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:06,813 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:A 2024-11-16T14:53:06,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:06,821 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bd5473650a054171bff8c3251c0e2f56_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bd5473650a054171bff8c3251c0e2f56_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:06,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/b6083403986449f5a27d56a6df653885, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:06,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/b6083403986449f5a27d56a6df653885 is 175, key is test_row_0/A:col10/1731768785460/Put/seqid=0 2024-11-16T14:53:06,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742223_1399 (size=31255) 2024-11-16T14:53:06,825 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=818, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/b6083403986449f5a27d56a6df653885 2024-11-16T14:53:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/1b8c646b026843b3bfc7da347ff66362 is 50, key is test_row_0/B:col10/1731768785460/Put/seqid=0 2024-11-16T14:53:06,836 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/b03a1b41dae446129cb5010e932b7cff as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/b03a1b41dae446129cb5010e932b7cff 2024-11-16T14:53:06,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742224_1400 (size=12301) 2024-11-16T14:53:06,840 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/C of 49b096ff4ffb696a771be33a33a5e082 into b03a1b41dae446129cb5010e932b7cff(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:06,840 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:06,840 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/C, priority=13, startTime=1731768786345; duration=0sec 2024-11-16T14:53:06,840 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:06,840 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:C 2024-11-16T14:53:06,843 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=818 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/1b8c646b026843b3bfc7da347ff66362 2024-11-16T14:53:06,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/591af84c890240739ba04a734b438f15 is 50, key is test_row_0/C:col10/1731768785460/Put/seqid=0 2024-11-16T14:53:06,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742225_1401 (size=12301) 2024-11-16T14:53:07,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768847000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768847000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768847000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768847001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:53:07,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47364 deadline: 1731768847060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,067 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in 49b096ff4ffb696a771be33a33a5e082 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=818 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/591af84c890240739ba04a734b438f15 2024-11-16T14:53:07,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/b6083403986449f5a27d56a6df653885 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b6083403986449f5a27d56a6df653885 2024-11-16T14:53:07,259 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b6083403986449f5a27d56a6df653885, entries=150, sequenceid=818, filesize=30.5 K 2024-11-16T14:53:07,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/1b8c646b026843b3bfc7da347ff66362 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/1b8c646b026843b3bfc7da347ff66362 2024-11-16T14:53:07,263 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/1b8c646b026843b3bfc7da347ff66362, entries=150, sequenceid=818, filesize=12.0 K 2024-11-16T14:53:07,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/591af84c890240739ba04a734b438f15 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/591af84c890240739ba04a734b438f15 2024-11-16T14:53:07,266 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/591af84c890240739ba04a734b438f15, entries=150, sequenceid=818, filesize=12.0 K 2024-11-16T14:53:07,267 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 49b096ff4ffb696a771be33a33a5e082 in 905ms, sequenceid=818, compaction requested=false 2024-11-16T14:53:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-16T14:53:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-16T14:53:07,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-16T14:53:07,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3700 sec 2024-11-16T14:53:07,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.3770 sec 2024-11-16T14:53:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:07,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T14:53:07,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:53:07,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:07,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:53:07,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:07,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:53:07,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:07,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160085a55eb6a04a448c6fcd3d0c190d59_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768787316/Put/seqid=0 2024-11-16T14:53:07,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742226_1402 (size=14994) 2024-11-16T14:53:07,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:07,333 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160085a55eb6a04a448c6fcd3d0c190d59_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160085a55eb6a04a448c6fcd3d0c190d59_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:07,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/b5bc32562cd0492da6dfaa42afe5b64c, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:07,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/b5bc32562cd0492da6dfaa42afe5b64c is 175, key is test_row_0/A:col10/1731768787316/Put/seqid=0 2024-11-16T14:53:07,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742227_1403 (size=39949) 2024-11-16T14:53:07,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768847342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768847342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768847342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768847346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768847461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768847461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 337 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768847462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768847478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768847680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768847680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 338 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768847691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768847691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,738 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=842, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/b5bc32562cd0492da6dfaa42afe5b64c 2024-11-16T14:53:07,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/a22a04e2d82d4d0cb4f434a1471d5047 is 50, key is test_row_0/B:col10/1731768787316/Put/seqid=0 2024-11-16T14:53:07,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742228_1404 (size=12301) 2024-11-16T14:53:07,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768847990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768847991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:08,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 339 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768848001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:08,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768848010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:08,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T14:53:08,041 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:08,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:08,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-16T14:53:08,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:53:08,042 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:08,043 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:08,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:08,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=842 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/a22a04e2d82d4d0cb4f434a1471d5047 2024-11-16T14:53:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:53:08,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/91f74f6dad08463ab3b14957aa345399 is 50, key is test_row_0/C:col10/1731768787316/Put/seqid=0 2024-11-16T14:53:08,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742229_1405 (size=12301) 2024-11-16T14:53:08,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T14:53:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:08,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:08,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T14:53:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:53:08,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T14:53:08,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:08,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47378 deadline: 1731768848500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:08,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47336 deadline: 1731768848501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:08,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 340 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47330 deadline: 1731768848520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:08,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1731768848530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:08,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:08,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:08,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=842 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/91f74f6dad08463ab3b14957aa345399 2024-11-16T14:53:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/b5bc32562cd0492da6dfaa42afe5b64c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b5bc32562cd0492da6dfaa42afe5b64c 2024-11-16T14:53:08,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b5bc32562cd0492da6dfaa42afe5b64c, entries=200, sequenceid=842, filesize=39.0 K 2024-11-16T14:53:08,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/a22a04e2d82d4d0cb4f434a1471d5047 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/a22a04e2d82d4d0cb4f434a1471d5047 2024-11-16T14:53:08,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/a22a04e2d82d4d0cb4f434a1471d5047, entries=150, sequenceid=842, filesize=12.0 K 2024-11-16T14:53:08,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/91f74f6dad08463ab3b14957aa345399 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/91f74f6dad08463ab3b14957aa345399 2024-11-16T14:53:08,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/91f74f6dad08463ab3b14957aa345399, entries=150, sequenceid=842, filesize=12.0 K 2024-11-16T14:53:08,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 49b096ff4ffb696a771be33a33a5e082 in 1278ms, sequenceid=842, compaction requested=true 2024-11-16T14:53:08,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:08,596 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:08,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:08,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:08,596 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:08,599 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:08,599 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/A is initiating minor compaction (all files) 2024-11-16T14:53:08,599 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/A in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49b096ff4ffb696a771be33a33a5e082:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:08,599 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/53cd520af3934dbc9e27a3f7f4a0c21c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b6083403986449f5a27d56a6df653885, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b5bc32562cd0492da6dfaa42afe5b64c] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=100.7 K 2024-11-16T14:53:08,599 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,599 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/53cd520af3934dbc9e27a3f7f4a0c21c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b6083403986449f5a27d56a6df653885, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b5bc32562cd0492da6dfaa42afe5b64c] 2024-11-16T14:53:08,600 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53cd520af3934dbc9e27a3f7f4a0c21c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=806, earliestPutTs=1731768784250 2024-11-16T14:53:08,600 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6083403986449f5a27d56a6df653885, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1731768785457 2024-11-16T14:53:08,600 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:08,600 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/B is initiating minor compaction (all files) 2024-11-16T14:53:08,600 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/B in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,600 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/fe012abc0c984c37a683d79ab23b606e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/1b8c646b026843b3bfc7da347ff66362, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/a22a04e2d82d4d0cb4f434a1471d5047] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=36.7 K 2024-11-16T14:53:08,600 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5bc32562cd0492da6dfaa42afe5b64c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=842, earliestPutTs=1731768786658 2024-11-16T14:53:08,601 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting fe012abc0c984c37a683d79ab23b606e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=806, earliestPutTs=1731768784250 2024-11-16T14:53:08,602 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8c646b026843b3bfc7da347ff66362, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1731768785457 2024-11-16T14:53:08,604 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting a22a04e2d82d4d0cb4f434a1471d5047, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=842, earliestPutTs=1731768786658 2024-11-16T14:53:08,612 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:08,628 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411163d5c143578c946c7a0a0dff546a50fde_49b096ff4ffb696a771be33a33a5e082 store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:08,631 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411163d5c143578c946c7a0a0dff546a50fde_49b096ff4ffb696a771be33a33a5e082, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:08,631 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163d5c143578c946c7a0a0dff546a50fde_49b096ff4ffb696a771be33a33a5e082 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:08,639 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#B#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:08,640 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/aaf23ef8090d4b779c86da1028861eb6 is 50, key is test_row_0/B:col10/1731768787316/Put/seqid=0 2024-11-16T14:53:08,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742230_1406 (size=4469) 2024-11-16T14:53:08,651 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#A#compaction#339 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:08,652 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/61484adfb5b641dea627fdd00f1f42f2 is 175, key is test_row_0/A:col10/1731768787316/Put/seqid=0 2024-11-16T14:53:08,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T14:53:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,655 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:53:08,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:53:08,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:08,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:53:08,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:08,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:53:08,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:53:08,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742231_1407 (size=13051) 2024-11-16T14:53:08,686 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/aaf23ef8090d4b779c86da1028861eb6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/aaf23ef8090d4b779c86da1028861eb6 2024-11-16T14:53:08,692 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/B of 49b096ff4ffb696a771be33a33a5e082 into aaf23ef8090d4b779c86da1028861eb6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:08,692 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:08,692 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/B, priority=13, startTime=1731768788596; duration=0sec 2024-11-16T14:53:08,692 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:08,692 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:B 2024-11-16T14:53:08,692 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:08,693 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:08,693 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): 49b096ff4ffb696a771be33a33a5e082/C is initiating minor compaction (all files) 2024-11-16T14:53:08,693 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49b096ff4ffb696a771be33a33a5e082/C in TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:08,693 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/b03a1b41dae446129cb5010e932b7cff, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/591af84c890240739ba04a734b438f15, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/91f74f6dad08463ab3b14957aa345399] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp, totalSize=36.7 K 2024-11-16T14:53:08,693 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting b03a1b41dae446129cb5010e932b7cff, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=806, earliestPutTs=1731768784250 2024-11-16T14:53:08,693 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 591af84c890240739ba04a734b438f15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1731768785457 2024-11-16T14:53:08,694 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 91f74f6dad08463ab3b14957aa345399, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=842, earliestPutTs=1731768786658 2024-11-16T14:53:08,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742232_1408 (size=32005) 2024-11-16T14:53:08,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d67014d3195e42f19e930c1f8a335b02_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768787342/Put/seqid=0 2024-11-16T14:53:08,702 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49b096ff4ffb696a771be33a33a5e082#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:08,702 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/e13d18dfe8b54b798d01500e7873c34f is 50, key is test_row_0/C:col10/1731768787316/Put/seqid=0 2024-11-16T14:53:08,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742233_1409 (size=12454) 2024-11-16T14:53:08,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742234_1410 (size=13051) 2024-11-16T14:53:08,744 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/e13d18dfe8b54b798d01500e7873c34f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/e13d18dfe8b54b798d01500e7873c34f 2024-11-16T14:53:08,750 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/C of 49b096ff4ffb696a771be33a33a5e082 into e13d18dfe8b54b798d01500e7873c34f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:08,750 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:08,750 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/C, priority=13, startTime=1731768788599; duration=0sec 2024-11-16T14:53:08,750 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:08,750 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:C 2024-11-16T14:53:09,102 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/61484adfb5b641dea627fdd00f1f42f2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/61484adfb5b641dea627fdd00f1f42f2 2024-11-16T14:53:09,105 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49b096ff4ffb696a771be33a33a5e082/A of 49b096ff4ffb696a771be33a33a5e082 into 61484adfb5b641dea627fdd00f1f42f2(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:09,105 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:09,105 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., storeName=49b096ff4ffb696a771be33a33a5e082/A, priority=13, startTime=1731768788596; duration=0sec 2024-11-16T14:53:09,105 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:09,105 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49b096ff4ffb696a771be33a33a5e082:A 2024-11-16T14:53:09,117 INFO [Thread-1468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1468. 2024-11-16T14:53:09,117 DEBUG [Thread-1468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:09,117 INFO [Thread-1470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1470. 2024-11-16T14:53:09,117 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,117 DEBUG [Thread-1470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:09,117 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,117 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:09,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:09,117 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,118 INFO [Thread-1474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1474. 2024-11-16T14:53:09,118 DEBUG [Thread-1474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:09,118 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:09,118 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,122 INFO [Thread-1476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1476. 2024-11-16T14:53:09,123 INFO [Thread-1472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1472. 2024-11-16T14:53:09,123 DEBUG [Thread-1476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:09,123 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,123 DEBUG [Thread-1472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:09,123 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,123 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,123 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:09,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:09,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:09,137 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d67014d3195e42f19e930c1f8a335b02_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d67014d3195e42f19e930c1f8a335b02_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:09,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/325b281672294cc6acadaf3d2ddc5a1f, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:09,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/325b281672294cc6acadaf3d2ddc5a1f is 175, key is test_row_0/A:col10/1731768787342/Put/seqid=0 2024-11-16T14:53:09,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742235_1411 (size=31255) 2024-11-16T14:53:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:53:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:09,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. as already flushing 2024-11-16T14:53:09,524 INFO [Thread-1459 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1459. 2024-11-16T14:53:09,525 DEBUG [Thread-1459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:09,525 DEBUG [Thread-1459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,525 DEBUG [Thread-1459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:09,534 INFO [Thread-1457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1457. 2024-11-16T14:53:09,534 DEBUG [Thread-1457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:09,534 DEBUG [Thread-1457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,534 DEBUG [Thread-1457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:09,543 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=853, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/325b281672294cc6acadaf3d2ddc5a1f 2024-11-16T14:53:09,545 INFO [Thread-1463 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1463. 2024-11-16T14:53:09,545 DEBUG [Thread-1463 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:09,545 DEBUG [Thread-1463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,545 DEBUG [Thread-1463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:09,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/cf876319896846bda97f42d4cbfb1c9e is 50, key is test_row_0/B:col10/1731768787342/Put/seqid=0 2024-11-16T14:53:09,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742236_1412 (size=12301) 2024-11-16T14:53:09,563 INFO [Thread-1461 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1461. 2024-11-16T14:53:09,563 DEBUG [Thread-1461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:09,563 DEBUG [Thread-1461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,563 DEBUG [Thread-1461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:09,563 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:09,958 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=853 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/cf876319896846bda97f42d4cbfb1c9e 2024-11-16T14:53:09,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/8c8d72d0e1fb485a8b68863d84862c51 is 50, key is test_row_0/C:col10/1731768787342/Put/seqid=0 2024-11-16T14:53:09,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742237_1413 (size=12301) 2024-11-16T14:53:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:53:10,368 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=853 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/8c8d72d0e1fb485a8b68863d84862c51 2024-11-16T14:53:10,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/325b281672294cc6acadaf3d2ddc5a1f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/325b281672294cc6acadaf3d2ddc5a1f 2024-11-16T14:53:10,376 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/325b281672294cc6acadaf3d2ddc5a1f, entries=150, sequenceid=853, filesize=30.5 K 2024-11-16T14:53:10,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/cf876319896846bda97f42d4cbfb1c9e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/cf876319896846bda97f42d4cbfb1c9e 2024-11-16T14:53:10,380 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/cf876319896846bda97f42d4cbfb1c9e, entries=150, sequenceid=853, filesize=12.0 K 2024-11-16T14:53:10,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/8c8d72d0e1fb485a8b68863d84862c51 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/8c8d72d0e1fb485a8b68863d84862c51 2024-11-16T14:53:10,385 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/8c8d72d0e1fb485a8b68863d84862c51, entries=150, sequenceid=853, filesize=12.0 K 2024-11-16T14:53:10,386 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=26.84 KB/27480 for 49b096ff4ffb696a771be33a33a5e082 in 1731ms, sequenceid=853, compaction requested=false 2024-11-16T14:53:10,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 49b096ff4ffb696a771be33a33a5e082: 2024-11-16T14:53:10,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:10,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-16T14:53:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-16T14:53:10,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-16T14:53:10,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3440 sec 2024-11-16T14:53:10,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.3480 sec 2024-11-16T14:53:11,710 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:53:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T14:53:12,211 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:16,424 WARN [regionserver/fc3fda29f660:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-16T14:53:16,427 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/ffc86e6d17f343b29d33a97450dc23f9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/5a2574b4723d4c9796cc1f7e050dbf4d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/29abd38a6c074d0396592de5b1419478, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a790b3afee494a7197ad7c6ff5827678, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/9753176bd923451fb947c66d58b6097f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9fb0a064b3d4539ab5a66dd297529f4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/82081788ac9042058f9d48f10428fbca, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a4563ebb52cb43a88484d7e4713747ff, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9bff21fdfe84642b43ac4320e31590a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f330dfabbd924377be339e19c45a8f16, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/7887bbdfc6b84ef0ab6979e6578ec137, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/0178370250634411bc9351edf05d842a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f423fa6df0dd4516810ebc440078fc89, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/45ef200c36844bebb4084886e7347510, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/cefdbfad5f384b999ba4d0db003ff5c1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/299da3fe48c848d59c3412b1906d76f4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/88c0cba22d5c40b6aadb0547da2159f6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/06a3f00d5934493aaef2aaee8d687595, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/4c3b611beacd4022b208af9921e47e5b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/53cd520af3934dbc9e27a3f7f4a0c21c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b6083403986449f5a27d56a6df653885, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b5bc32562cd0492da6dfaa42afe5b64c] to archive 2024-11-16T14:53:16,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:53:16,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/ffc86e6d17f343b29d33a97450dc23f9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/ffc86e6d17f343b29d33a97450dc23f9 2024-11-16T14:53:16,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/5a2574b4723d4c9796cc1f7e050dbf4d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/5a2574b4723d4c9796cc1f7e050dbf4d 2024-11-16T14:53:16,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/29abd38a6c074d0396592de5b1419478 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/29abd38a6c074d0396592de5b1419478 2024-11-16T14:53:16,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a790b3afee494a7197ad7c6ff5827678 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a790b3afee494a7197ad7c6ff5827678 2024-11-16T14:53:16,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/9753176bd923451fb947c66d58b6097f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/9753176bd923451fb947c66d58b6097f 2024-11-16T14:53:16,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9fb0a064b3d4539ab5a66dd297529f4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9fb0a064b3d4539ab5a66dd297529f4 2024-11-16T14:53:16,434 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/82081788ac9042058f9d48f10428fbca to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/82081788ac9042058f9d48f10428fbca 2024-11-16T14:53:16,435 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a4563ebb52cb43a88484d7e4713747ff to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/a4563ebb52cb43a88484d7e4713747ff 2024-11-16T14:53:16,436 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9bff21fdfe84642b43ac4320e31590a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/c9bff21fdfe84642b43ac4320e31590a 2024-11-16T14:53:16,436 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f330dfabbd924377be339e19c45a8f16 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f330dfabbd924377be339e19c45a8f16 2024-11-16T14:53:16,437 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/7887bbdfc6b84ef0ab6979e6578ec137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/7887bbdfc6b84ef0ab6979e6578ec137 2024-11-16T14:53:16,438 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/0178370250634411bc9351edf05d842a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/0178370250634411bc9351edf05d842a 2024-11-16T14:53:16,438 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f423fa6df0dd4516810ebc440078fc89 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/f423fa6df0dd4516810ebc440078fc89 2024-11-16T14:53:16,439 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/45ef200c36844bebb4084886e7347510 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/45ef200c36844bebb4084886e7347510 2024-11-16T14:53:16,440 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/cefdbfad5f384b999ba4d0db003ff5c1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/cefdbfad5f384b999ba4d0db003ff5c1 2024-11-16T14:53:16,441 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/299da3fe48c848d59c3412b1906d76f4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/299da3fe48c848d59c3412b1906d76f4 2024-11-16T14:53:16,442 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/88c0cba22d5c40b6aadb0547da2159f6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/88c0cba22d5c40b6aadb0547da2159f6 2024-11-16T14:53:16,442 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/06a3f00d5934493aaef2aaee8d687595 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/06a3f00d5934493aaef2aaee8d687595 2024-11-16T14:53:16,443 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/4c3b611beacd4022b208af9921e47e5b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/4c3b611beacd4022b208af9921e47e5b 2024-11-16T14:53:16,444 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/53cd520af3934dbc9e27a3f7f4a0c21c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/53cd520af3934dbc9e27a3f7f4a0c21c 2024-11-16T14:53:16,445 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b6083403986449f5a27d56a6df653885 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b6083403986449f5a27d56a6df653885 2024-11-16T14:53:16,445 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b5bc32562cd0492da6dfaa42afe5b64c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/b5bc32562cd0492da6dfaa42afe5b64c 2024-11-16T14:53:16,449 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/eb7bfa49003245dc9dbff3b227c21133, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/0b7e33ce84054a10a0ba9bebf3a42e45, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/e6b867d8e97e4460aae743085c35dccd, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c4bca69742804ee9aa45b39606eb15b2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f430c2d721a94701b882438c6ebe45be, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/3a6ca68ff3974f01abdd675c4d9dcbc4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6cb051435916468fbdbb25b1c896d4a2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5e50a4b2a12241cb903b40022302dad5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/2798e09c8f7c43418e456dfe8aedb4b8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c35fa0741e8e49fc9785ac2cfb6b0ffb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6bb56529684a440a92e0342206c5fc7e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca43bbc13c324c8d881542b7e2bb34e4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca6e44bc431f415481ec9888302fdb76, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/20af16ba8bd6477b98ea2e7998d00cbc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f11b002ce34d4cb2b54c5fbbdff99785, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5b50aef6f61f4799b39949231393a9f0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/543c249f9a25434ebef8af3399e8eba7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/42d1325b30d64c6cb710c4df4c11e7a9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/fe012abc0c984c37a683d79ab23b606e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/cfc00911482d427ab1eb221832cece5f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/1b8c646b026843b3bfc7da347ff66362, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/a22a04e2d82d4d0cb4f434a1471d5047] to archive 2024-11-16T14:53:16,450 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:53:16,451 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/eb7bfa49003245dc9dbff3b227c21133 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/eb7bfa49003245dc9dbff3b227c21133 2024-11-16T14:53:16,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/0b7e33ce84054a10a0ba9bebf3a42e45 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/0b7e33ce84054a10a0ba9bebf3a42e45 2024-11-16T14:53:16,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/e6b867d8e97e4460aae743085c35dccd to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/e6b867d8e97e4460aae743085c35dccd 2024-11-16T14:53:16,453 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c4bca69742804ee9aa45b39606eb15b2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c4bca69742804ee9aa45b39606eb15b2 2024-11-16T14:53:16,454 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f430c2d721a94701b882438c6ebe45be to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f430c2d721a94701b882438c6ebe45be 2024-11-16T14:53:16,454 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/3a6ca68ff3974f01abdd675c4d9dcbc4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/3a6ca68ff3974f01abdd675c4d9dcbc4 2024-11-16T14:53:16,455 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6cb051435916468fbdbb25b1c896d4a2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6cb051435916468fbdbb25b1c896d4a2 2024-11-16T14:53:16,456 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5e50a4b2a12241cb903b40022302dad5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5e50a4b2a12241cb903b40022302dad5 2024-11-16T14:53:16,456 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/2798e09c8f7c43418e456dfe8aedb4b8 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/2798e09c8f7c43418e456dfe8aedb4b8 2024-11-16T14:53:16,457 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c35fa0741e8e49fc9785ac2cfb6b0ffb to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/c35fa0741e8e49fc9785ac2cfb6b0ffb 2024-11-16T14:53:16,458 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6bb56529684a440a92e0342206c5fc7e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/6bb56529684a440a92e0342206c5fc7e 2024-11-16T14:53:16,458 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca43bbc13c324c8d881542b7e2bb34e4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca43bbc13c324c8d881542b7e2bb34e4 2024-11-16T14:53:16,459 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca6e44bc431f415481ec9888302fdb76 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/ca6e44bc431f415481ec9888302fdb76 2024-11-16T14:53:16,459 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/20af16ba8bd6477b98ea2e7998d00cbc to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/20af16ba8bd6477b98ea2e7998d00cbc 2024-11-16T14:53:16,460 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f11b002ce34d4cb2b54c5fbbdff99785 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/f11b002ce34d4cb2b54c5fbbdff99785 2024-11-16T14:53:16,461 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5b50aef6f61f4799b39949231393a9f0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/5b50aef6f61f4799b39949231393a9f0 2024-11-16T14:53:16,461 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/543c249f9a25434ebef8af3399e8eba7 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/543c249f9a25434ebef8af3399e8eba7 2024-11-16T14:53:16,462 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/42d1325b30d64c6cb710c4df4c11e7a9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/42d1325b30d64c6cb710c4df4c11e7a9 2024-11-16T14:53:16,462 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/fe012abc0c984c37a683d79ab23b606e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/fe012abc0c984c37a683d79ab23b606e 2024-11-16T14:53:16,463 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/cfc00911482d427ab1eb221832cece5f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/cfc00911482d427ab1eb221832cece5f 2024-11-16T14:53:16,464 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/1b8c646b026843b3bfc7da347ff66362 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/1b8c646b026843b3bfc7da347ff66362 2024-11-16T14:53:16,464 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/a22a04e2d82d4d0cb4f434a1471d5047 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/a22a04e2d82d4d0cb4f434a1471d5047 2024-11-16T14:53:16,467 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/53116770e74d496e9a6021a6e00cdb09, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/962150132adb42f59d5a151e5d80f264, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/790543cc33be4e2fa511743df399864e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f5769305bf994f1bbd8ed1a1834e222b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f33dabf11d6d464682c09c3c4ce91b11, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a5c1fcb9000b4094a4366a8c11b1c655, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f2a7e22d505b40128a4ef36e12658f67, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/7bb111f91f804e25850bd5244af6fbcb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/909d0c2815884013b6e2e3b7428e138e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/02a96af20155429a802cd6620c1a0bd1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/d67a3302ea5b437689c54afb2a778a67, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/60b6f50175f54a50bc99928736ddb776, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/9b2c94192501402aad7f04118beb53ac, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/bfedfc03376a4d3abac6c973cc78cdf6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a9415e32074c47c1ba1a303446034434, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4e2a3d1a0e284362b17ea14d83aca73b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4cdbca37413540d08850a616d692e901, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/1af28fc863d242688a51d44c1dff1ad5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/b03a1b41dae446129cb5010e932b7cff, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/e5cc0d76ff284191935a09c3db830dd3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/591af84c890240739ba04a734b438f15, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/91f74f6dad08463ab3b14957aa345399] to archive 2024-11-16T14:53:16,467 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:53:16,468 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/53116770e74d496e9a6021a6e00cdb09 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/53116770e74d496e9a6021a6e00cdb09 2024-11-16T14:53:16,469 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/962150132adb42f59d5a151e5d80f264 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/962150132adb42f59d5a151e5d80f264 2024-11-16T14:53:16,470 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/790543cc33be4e2fa511743df399864e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/790543cc33be4e2fa511743df399864e 2024-11-16T14:53:16,471 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f5769305bf994f1bbd8ed1a1834e222b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f5769305bf994f1bbd8ed1a1834e222b 2024-11-16T14:53:16,472 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f33dabf11d6d464682c09c3c4ce91b11 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f33dabf11d6d464682c09c3c4ce91b11 2024-11-16T14:53:16,473 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a5c1fcb9000b4094a4366a8c11b1c655 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a5c1fcb9000b4094a4366a8c11b1c655 2024-11-16T14:53:16,474 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f2a7e22d505b40128a4ef36e12658f67 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/f2a7e22d505b40128a4ef36e12658f67 2024-11-16T14:53:16,475 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/7bb111f91f804e25850bd5244af6fbcb to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/7bb111f91f804e25850bd5244af6fbcb 2024-11-16T14:53:16,476 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/909d0c2815884013b6e2e3b7428e138e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/909d0c2815884013b6e2e3b7428e138e 2024-11-16T14:53:16,477 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/02a96af20155429a802cd6620c1a0bd1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/02a96af20155429a802cd6620c1a0bd1 2024-11-16T14:53:16,477 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/d67a3302ea5b437689c54afb2a778a67 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/d67a3302ea5b437689c54afb2a778a67 2024-11-16T14:53:16,478 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/60b6f50175f54a50bc99928736ddb776 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/60b6f50175f54a50bc99928736ddb776 2024-11-16T14:53:16,479 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/9b2c94192501402aad7f04118beb53ac to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/9b2c94192501402aad7f04118beb53ac 2024-11-16T14:53:16,480 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/bfedfc03376a4d3abac6c973cc78cdf6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/bfedfc03376a4d3abac6c973cc78cdf6 2024-11-16T14:53:16,481 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a9415e32074c47c1ba1a303446034434 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/a9415e32074c47c1ba1a303446034434 2024-11-16T14:53:16,482 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4e2a3d1a0e284362b17ea14d83aca73b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4e2a3d1a0e284362b17ea14d83aca73b 2024-11-16T14:53:16,482 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4cdbca37413540d08850a616d692e901 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/4cdbca37413540d08850a616d692e901 2024-11-16T14:53:16,483 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/1af28fc863d242688a51d44c1dff1ad5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/1af28fc863d242688a51d44c1dff1ad5 2024-11-16T14:53:16,484 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/b03a1b41dae446129cb5010e932b7cff to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/b03a1b41dae446129cb5010e932b7cff 2024-11-16T14:53:16,485 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/e5cc0d76ff284191935a09c3db830dd3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/e5cc0d76ff284191935a09c3db830dd3 2024-11-16T14:53:16,486 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/591af84c890240739ba04a734b438f15 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/591af84c890240739ba04a734b438f15 2024-11-16T14:53:16,487 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/91f74f6dad08463ab3b14957aa345399 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/91f74f6dad08463ab3b14957aa345399 2024-11-16T14:53:17,163 INFO [Thread-1465 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1465. 2024-11-16T14:53:17,163 DEBUG [Thread-1465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:17,164 DEBUG [Thread-1465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:17,164 DEBUG [Thread-1465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:17,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 271 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 287 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 920 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2760 rows 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 923 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2769 rows 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 914 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2742 rows 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 919 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2757 rows 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 920 2024-11-16T14:53:17,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2760 rows 2024-11-16T14:53:17,165 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:53:17,165 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:53:17,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:17,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:17,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:17,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:53:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:53:17,169 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768797168"}]},"ts":"1731768797168"} 2024-11-16T14:53:17,169 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:53:17,170 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:53:17,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:53:17,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, UNASSIGN}] 2024-11-16T14:53:17,172 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, UNASSIGN 2024-11-16T14:53:17,173 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=49b096ff4ffb696a771be33a33a5e082, regionState=CLOSING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:17,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, UNASSIGN because future has completed 2024-11-16T14:53:17,174 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:53:17,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:53:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:53:17,326 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:17,326 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:53:17,326 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 49b096ff4ffb696a771be33a33a5e082, disabling compactions & flushes 2024-11-16T14:53:17,326 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:17,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:17,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. after waiting 0 ms 2024-11-16T14:53:17,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:17,327 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 49b096ff4ffb696a771be33a33a5e082 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T14:53:17,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=A 2024-11-16T14:53:17,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:17,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=B 2024-11-16T14:53:17,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:17,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49b096ff4ffb696a771be33a33a5e082, store=C 2024-11-16T14:53:17,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:17,334 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111645dff6e4cad14c2bbe712b60d9350ca8_49b096ff4ffb696a771be33a33a5e082 is 50, key is test_row_0/A:col10/1731768797161/Put/seqid=0 2024-11-16T14:53:17,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742238_1414 (size=12454) 2024-11-16T14:53:17,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:53:17,740 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:17,744 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111645dff6e4cad14c2bbe712b60d9350ca8_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111645dff6e4cad14c2bbe712b60d9350ca8_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:17,745 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/734af928c545440f875744d2f5bd4419, store: [table=TestAcidGuarantees family=A region=49b096ff4ffb696a771be33a33a5e082] 2024-11-16T14:53:17,745 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/734af928c545440f875744d2f5bd4419 is 175, key is test_row_0/A:col10/1731768797161/Put/seqid=0 2024-11-16T14:53:17,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742239_1415 (size=31255) 2024-11-16T14:53:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:53:18,151 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=864, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/734af928c545440f875744d2f5bd4419 2024-11-16T14:53:18,163 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/2d019e75eab04e54b3417b7b82a2d2cd is 50, key is test_row_0/B:col10/1731768797161/Put/seqid=0 2024-11-16T14:53:18,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742240_1416 (size=12301) 2024-11-16T14:53:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:53:18,567 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=864 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/2d019e75eab04e54b3417b7b82a2d2cd 2024-11-16T14:53:18,574 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/b7ae8fab59384ec3b0d4f94216b482d1 is 50, key is test_row_0/C:col10/1731768797161/Put/seqid=0 2024-11-16T14:53:18,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742241_1417 (size=12301) 2024-11-16T14:53:18,579 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=864 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/b7ae8fab59384ec3b0d4f94216b482d1 2024-11-16T14:53:18,584 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/A/734af928c545440f875744d2f5bd4419 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/734af928c545440f875744d2f5bd4419 2024-11-16T14:53:18,589 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/734af928c545440f875744d2f5bd4419, entries=150, sequenceid=864, filesize=30.5 K 2024-11-16T14:53:18,591 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/B/2d019e75eab04e54b3417b7b82a2d2cd as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/2d019e75eab04e54b3417b7b82a2d2cd 2024-11-16T14:53:18,595 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/2d019e75eab04e54b3417b7b82a2d2cd, entries=150, sequenceid=864, filesize=12.0 K 2024-11-16T14:53:18,596 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/.tmp/C/b7ae8fab59384ec3b0d4f94216b482d1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/b7ae8fab59384ec3b0d4f94216b482d1 2024-11-16T14:53:18,600 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/b7ae8fab59384ec3b0d4f94216b482d1, entries=150, sequenceid=864, filesize=12.0 K 2024-11-16T14:53:18,600 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 49b096ff4ffb696a771be33a33a5e082 in 1273ms, sequenceid=864, compaction requested=true 2024-11-16T14:53:18,611 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/recovered.edits/867.seqid, newMaxSeqId=867, maxSeqId=4 2024-11-16T14:53:18,612 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. 2024-11-16T14:53:18,612 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 49b096ff4ffb696a771be33a33a5e082: Waiting for close lock at 1731768797326Running coprocessor pre-close hooks at 1731768797326Disabling compacts and flushes for region at 1731768797326Disabling writes for close at 1731768797327 (+1 ms)Obtaining lock to block concurrent updates at 1731768797327Preparing flush snapshotting stores in 49b096ff4ffb696a771be33a33a5e082 at 1731768797327Finished memstore snapshotting TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731768797327Flushing stores of TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082. at 1731768797328 (+1 ms)Flushing 49b096ff4ffb696a771be33a33a5e082/A: creating writer at 1731768797328Flushing mob file 49b096ff4ffb696a771be33a33a5e082/A: appending metadata at 1731768797334 (+6 ms)Flushing mob file 49b096ff4ffb696a771be33a33a5e082/A: closing flushed file at 1731768797334Flushing 49b096ff4ffb696a771be33a33a5e082/A: appending metadata at 1731768797745 (+411 ms)Flushing 49b096ff4ffb696a771be33a33a5e082/A: closing flushed file at 1731768797745Flushing 49b096ff4ffb696a771be33a33a5e082/B: creating writer at 1731768798159 (+414 ms)Flushing 49b096ff4ffb696a771be33a33a5e082/B: appending metadata at 1731768798163 (+4 ms)Flushing 49b096ff4ffb696a771be33a33a5e082/B: closing flushed file at 1731768798163Flushing 49b096ff4ffb696a771be33a33a5e082/C: creating writer at 1731768798571 (+408 ms)Flushing 49b096ff4ffb696a771be33a33a5e082/C: appending metadata at 1731768798574 (+3 ms)Flushing 49b096ff4ffb696a771be33a33a5e082/C: closing flushed file at 1731768798574Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7986b8ba: reopening flushed file at 1731768798583 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49138d8e: reopening flushed file at 1731768798590 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@498f07d0: reopening flushed file at 1731768798595 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 49b096ff4ffb696a771be33a33a5e082 in 1273ms, sequenceid=864, compaction requested=true at 1731768798600 (+5 ms)Writing region close event to WAL at 1731768798607 (+7 ms)Running coprocessor post-close hooks at 1731768798612 (+5 ms)Closed at 1731768798612 2024-11-16T14:53:18,614 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:18,614 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=49b096ff4ffb696a771be33a33a5e082, regionState=CLOSED 2024-11-16T14:53:18,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:53:18,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-16T14:53:18,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 49b096ff4ffb696a771be33a33a5e082, server=fc3fda29f660,43955,1731768673386 in 1.4420 sec 2024-11-16T14:53:18,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-16T14:53:18,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49b096ff4ffb696a771be33a33a5e082, UNASSIGN in 1.4480 sec 2024-11-16T14:53:18,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-16T14:53:18,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4520 sec 2024-11-16T14:53:18,626 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768798625"}]},"ts":"1731768798625"} 2024-11-16T14:53:18,627 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:53:18,627 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:53:18,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4620 sec 2024-11-16T14:53:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T14:53:19,311 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:19,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:53:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:19,314 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T14:53:19,315 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:19,318 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,322 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/recovered.edits] 2024-11-16T14:53:19,327 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/325b281672294cc6acadaf3d2ddc5a1f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/325b281672294cc6acadaf3d2ddc5a1f 2024-11-16T14:53:19,329 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/61484adfb5b641dea627fdd00f1f42f2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/61484adfb5b641dea627fdd00f1f42f2 2024-11-16T14:53:19,331 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/734af928c545440f875744d2f5bd4419 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/A/734af928c545440f875744d2f5bd4419 2024-11-16T14:53:19,335 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/2d019e75eab04e54b3417b7b82a2d2cd to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/2d019e75eab04e54b3417b7b82a2d2cd 2024-11-16T14:53:19,336 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/aaf23ef8090d4b779c86da1028861eb6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/aaf23ef8090d4b779c86da1028861eb6 2024-11-16T14:53:19,338 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/cf876319896846bda97f42d4cbfb1c9e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/B/cf876319896846bda97f42d4cbfb1c9e 2024-11-16T14:53:19,342 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/8c8d72d0e1fb485a8b68863d84862c51 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/8c8d72d0e1fb485a8b68863d84862c51 2024-11-16T14:53:19,344 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/b7ae8fab59384ec3b0d4f94216b482d1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/b7ae8fab59384ec3b0d4f94216b482d1 2024-11-16T14:53:19,345 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/e13d18dfe8b54b798d01500e7873c34f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/C/e13d18dfe8b54b798d01500e7873c34f 2024-11-16T14:53:19,349 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/recovered.edits/867.seqid to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082/recovered.edits/867.seqid 2024-11-16T14:53:19,349 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,349 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:53:19,350 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:53:19,351 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T14:53:19,356 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160085a55eb6a04a448c6fcd3d0c190d59_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160085a55eb6a04a448c6fcd3d0c190d59_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,358 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162b985921a22f4ce7851ff9b52ee2fb50_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162b985921a22f4ce7851ff9b52ee2fb50_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,360 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111638e291620e924d66b68a00d9573695eb_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111638e291620e924d66b68a00d9573695eb_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,361 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164062ca75380f46ab8ab5836954a81e51_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164062ca75380f46ab8ab5836954a81e51_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,363 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111645dff6e4cad14c2bbe712b60d9350ca8_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111645dff6e4cad14c2bbe712b60d9350ca8_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,364 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164dfdaf4a126246bcbb9b9d4e956a2e26_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164dfdaf4a126246bcbb9b9d4e956a2e26_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,365 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116511c49adaed149e7951e63c88db6ed1f_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116511c49adaed149e7951e63c88db6ed1f_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,366 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116542fcac8ed9f43cf8273be8bfff98d15_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116542fcac8ed9f43cf8273be8bfff98d15_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,367 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165cfd5a595cf44440b513907191819d50_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165cfd5a595cf44440b513907191819d50_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,368 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116830502e866804162b8bcf370bff88689_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116830502e866804162b8bcf370bff88689_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,368 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a1fe4123617e4fdba08e907c810b3c41_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a1fe4123617e4fdba08e907c810b3c41_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,369 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a6701a20c26a4585930eca543b04a8af_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a6701a20c26a4585930eca543b04a8af_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,370 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b901896062f94aa982784ea9e497ecfa_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b901896062f94aa982784ea9e497ecfa_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,371 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bd5473650a054171bff8c3251c0e2f56_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bd5473650a054171bff8c3251c0e2f56_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,371 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ca4b259a5a674599bc6122f6bcca25d4_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ca4b259a5a674599bc6122f6bcca25d4_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,372 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d67014d3195e42f19e930c1f8a335b02_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d67014d3195e42f19e930c1f8a335b02_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,373 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eb2dc75287894b04a5cd7a6feaa9d5f2_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eb2dc75287894b04a5cd7a6feaa9d5f2_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,374 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fae297812e18487f8e17a75e9be191f3_49b096ff4ffb696a771be33a33a5e082 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fae297812e18487f8e17a75e9be191f3_49b096ff4ffb696a771be33a33a5e082 2024-11-16T14:53:19,374 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:53:19,376 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:19,377 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:53:19,379 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:53:19,380 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:19,380 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:53:19,380 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731768799380"}]},"ts":"9223372036854775807"} 2024-11-16T14:53:19,381 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:53:19,381 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 49b096ff4ffb696a771be33a33a5e082, NAME => 'TestAcidGuarantees,,1731768765917.49b096ff4ffb696a771be33a33a5e082.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:53:19,381 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:53:19,381 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731768799381"}]},"ts":"9223372036854775807"} 2024-11-16T14:53:19,383 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:53:19,383 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:19,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-16T14:53:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T14:53:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:53:19,421 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:19,430 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=503 (was 460) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5563 (was 4845) - AvailableMemoryMB LEAK? - 2024-11-16T14:53:19,439 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=503, ProcessCount=11, AvailableMemoryMB=5562 2024-11-16T14:53:19,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:53:19,441 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:53:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:19,442 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:53:19,442 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:19,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-16T14:53:19,443 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:53:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:53:19,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742242_1418 (size=1061) 2024-11-16T14:53:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:53:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:53:19,853 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d 2024-11-16T14:53:19,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742243_1419 (size=53) 2024-11-16T14:53:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:53:20,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:53:20,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d96afd6d0e8810c274529387dcd006b9, disabling compactions & flushes 2024-11-16T14:53:20,264 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:20,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:20,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. after waiting 0 ms 2024-11-16T14:53:20,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:20,264 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:20,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d96afd6d0e8810c274529387dcd006b9: Waiting for close lock at 1731768800264Disabling compacts and flushes for region at 1731768800264Disabling writes for close at 1731768800264Writing region close event to WAL at 1731768800264Closed at 1731768800264 2024-11-16T14:53:20,266 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:53:20,267 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731768800266"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731768800266"}]},"ts":"1731768800266"} 2024-11-16T14:53:20,269 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:53:20,270 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:53:20,271 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768800271"}]},"ts":"1731768800271"} 2024-11-16T14:53:20,273 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:53:20,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d96afd6d0e8810c274529387dcd006b9, ASSIGN}] 2024-11-16T14:53:20,275 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d96afd6d0e8810c274529387dcd006b9, ASSIGN 2024-11-16T14:53:20,276 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d96afd6d0e8810c274529387dcd006b9, ASSIGN; state=OFFLINE, location=fc3fda29f660,43955,1731768673386; forceNewPlan=false, retain=false 2024-11-16T14:53:20,426 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=d96afd6d0e8810c274529387dcd006b9, regionState=OPENING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:20,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d96afd6d0e8810c274529387dcd006b9, ASSIGN because future has completed 2024-11-16T14:53:20,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:53:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:53:20,583 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:20,583 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:53:20,584 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,584 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:53:20,585 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,585 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,586 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,588 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:53:20,588 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d96afd6d0e8810c274529387dcd006b9 columnFamilyName A 2024-11-16T14:53:20,588 DEBUG [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:20,589 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] regionserver.HStore(327): Store=d96afd6d0e8810c274529387dcd006b9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:53:20,589 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,590 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:53:20,590 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d96afd6d0e8810c274529387dcd006b9 columnFamilyName B 2024-11-16T14:53:20,591 DEBUG [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:20,591 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] regionserver.HStore(327): Store=d96afd6d0e8810c274529387dcd006b9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:53:20,591 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,593 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:53:20,593 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d96afd6d0e8810c274529387dcd006b9 columnFamilyName C 2024-11-16T14:53:20,593 DEBUG [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:20,593 INFO [StoreOpener-d96afd6d0e8810c274529387dcd006b9-1 {}] regionserver.HStore(327): Store=d96afd6d0e8810c274529387dcd006b9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:53:20,594 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:20,594 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,594 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,594 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,595 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,595 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,595 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:53:20,596 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,598 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:53:20,598 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened d96afd6d0e8810c274529387dcd006b9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59136655, jitterRate=-0.11879517138004303}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:53:20,598 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:20,599 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for d96afd6d0e8810c274529387dcd006b9: Running coprocessor pre-open hook at 1731768800585Writing region info on filesystem at 1731768800585Initializing all the Stores at 1731768800586 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768800586Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768800586Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768800586Cleaning up temporary data from old regions at 1731768800595 (+9 ms)Running coprocessor post-open hooks at 1731768800598 (+3 ms)Region opened successfully at 1731768800599 (+1 ms) 2024-11-16T14:53:20,600 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., pid=116, masterSystemTime=1731768800580 2024-11-16T14:53:20,601 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:20,601 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:20,602 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=d96afd6d0e8810c274529387dcd006b9, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:20,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:53:20,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-16T14:53:20,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 in 175 msec 2024-11-16T14:53:20,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-16T14:53:20,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d96afd6d0e8810c274529387dcd006b9, ASSIGN in 332 msec 2024-11-16T14:53:20,607 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:53:20,607 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768800607"}]},"ts":"1731768800607"} 2024-11-16T14:53:20,608 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:53:20,609 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:53:20,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-16T14:53:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-16T14:53:21,591 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:21,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca175cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,595 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65582560, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,597 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46207622, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,598 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,601 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:53:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fc3fda29f660,35823,1731768672605 2024-11-16T14:53:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38fab50f 2024-11-16T14:53:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:53:21,603 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:53:21,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491481f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,606 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626d65e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,607 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ea131ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7631696d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,610 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398ce6ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,611 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71782864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126b8236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,614 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677698f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,615 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531473c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea00808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,617 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c1fe71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,618 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78217593, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1589e05f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,620 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11642ce4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,621 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538d9a50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c19380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,623 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6caac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d4c757f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b23f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,627 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19c57a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fc1ebd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7d791a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,631 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d6e3a18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,632 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac77f06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b3a94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,640 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2891767e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,641 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12fa2819, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c68a1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:21,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:21,642 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fae858, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:21,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a12afe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:21,644 DEBUG [Thread-1901 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,644 DEBUG [Thread-1910 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,644 DEBUG [Thread-1899 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,645 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,645 DEBUG [Thread-1908 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,645 DEBUG [Thread-1914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,646 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,647 DEBUG [Thread-1906 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,648 DEBUG [Thread-1912 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-16T14:53:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:53:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:53:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:53:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,650 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:53:21,651 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:21,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:21,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:53:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:53:21,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:53:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:53:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:53:21,655 DEBUG [Thread-1895 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:21,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:53:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:53:21,661 DEBUG [Thread-1897 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:21,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:21,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2] 2024-11-16T14:53:21,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/31aaef3bddc8415a97f6c5aedb58a261 is 50, key is test_row_0/A:col10/1731768801656/Put/seqid=0 2024-11-16T14:53:21,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742244_1420 (size=12001) 2024-11-16T14:53:21,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768861690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:21,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768801669, max timestamp=1731768801701Num uniques 859; 2024-11-16T14:53:21,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:53:21,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768801669, max timestamp=1731768801701Num uniques 859; 2024-11-16T14:53:21,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:53:21,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768801669, max timestamp=1731768801701Num uniques 859; 2024-11-16T14:53:21,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:53:21,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768861731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:21,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768861733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:21,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:21,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768861738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:21,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768861739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:53:21,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T14:53:21,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:21,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:21,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:21,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:21,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:21,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768861811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:21,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768861840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:21,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768861850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768861850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:21,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768861860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:21,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T14:53:21,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:21,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:21,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:21,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:21,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:53:22,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768862030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768862061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768862061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768862061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768862070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/31aaef3bddc8415a97f6c5aedb58a261 2024-11-16T14:53:22,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/7bfbfadc4125499d8926a019ae27157b is 50, key is test_row_0/B:col10/1731768801656/Put/seqid=0 2024-11-16T14:53:22,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T14:53:22,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:22,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:22,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:22,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:22,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:22,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742245_1421 (size=12001) 2024-11-16T14:53:22,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/7bfbfadc4125499d8926a019ae27157b 2024-11-16T14:53:22,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/e403ae56e9f64d1eadd27eeef136d19c is 50, key is test_row_0/C:col10/1731768801656/Put/seqid=0 2024-11-16T14:53:22,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742246_1422 (size=12001) 2024-11-16T14:53:22,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/e403ae56e9f64d1eadd27eeef136d19c 2024-11-16T14:53:22,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/31aaef3bddc8415a97f6c5aedb58a261 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/31aaef3bddc8415a97f6c5aedb58a261 2024-11-16T14:53:22,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/31aaef3bddc8415a97f6c5aedb58a261, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T14:53:22,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/7bfbfadc4125499d8926a019ae27157b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7bfbfadc4125499d8926a019ae27157b 2024-11-16T14:53:22,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7bfbfadc4125499d8926a019ae27157b, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T14:53:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/e403ae56e9f64d1eadd27eeef136d19c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/e403ae56e9f64d1eadd27eeef136d19c 2024-11-16T14:53:22,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/e403ae56e9f64d1eadd27eeef136d19c, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T14:53:22,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d96afd6d0e8810c274529387dcd006b9 in 502ms, sequenceid=11, compaction requested=false 2024-11-16T14:53:22,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:22,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T14:53:22,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:22,265 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T14:53:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:22,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/d05a6681007f4fb28ca6171e22cb5332 is 50, key is test_row_0/A:col10/1731768801737/Put/seqid=0 2024-11-16T14:53:22,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742247_1423 (size=12001) 2024-11-16T14:53:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:53:22,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:22,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768862370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768862377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768862381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768862382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768862390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768862490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768862490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768862501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,675 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/d05a6681007f4fb28ca6171e22cb5332 2024-11-16T14:53:22,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768862700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768862711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/f8ceb3780db6409fbacefa80b0cb25f9 is 50, key is test_row_0/B:col10/1731768801737/Put/seqid=0 2024-11-16T14:53:22,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742248_1424 (size=12001) 2024-11-16T14:53:22,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768862721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:53:22,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768862890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:22,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768862911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768863021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768863021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768863030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,120 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/f8ceb3780db6409fbacefa80b0cb25f9 2024-11-16T14:53:23,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/d8c13af278cc4ed4a65c5cea8bd99342 is 50, key is test_row_0/C:col10/1731768801737/Put/seqid=0 2024-11-16T14:53:23,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742249_1425 (size=12001) 2024-11-16T14:53:23,530 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/d8c13af278cc4ed4a65c5cea8bd99342 2024-11-16T14:53:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/d05a6681007f4fb28ca6171e22cb5332 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d05a6681007f4fb28ca6171e22cb5332 2024-11-16T14:53:23,536 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d05a6681007f4fb28ca6171e22cb5332, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T14:53:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/f8ceb3780db6409fbacefa80b0cb25f9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/f8ceb3780db6409fbacefa80b0cb25f9 2024-11-16T14:53:23,540 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/f8ceb3780db6409fbacefa80b0cb25f9, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T14:53:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/d8c13af278cc4ed4a65c5cea8bd99342 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d8c13af278cc4ed4a65c5cea8bd99342 2024-11-16T14:53:23,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768863531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,543 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d8c13af278cc4ed4a65c5cea8bd99342, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T14:53:23,544 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for d96afd6d0e8810c274529387dcd006b9 in 1280ms, sequenceid=46, compaction requested=false 2024-11-16T14:53:23,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:23,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:23,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-16T14:53:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-16T14:53:23,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-16T14:53:23,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8940 sec 2024-11-16T14:53:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:23,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:53:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:23,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.9020 sec 2024-11-16T14:53:23,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/16b5099496c24c6387bcd3248fddb37b is 50, key is test_row_0/A:col10/1731768802367/Put/seqid=0 2024-11-16T14:53:23,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742250_1426 (size=14341) 2024-11-16T14:53:23,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/16b5099496c24c6387bcd3248fddb37b 2024-11-16T14:53:23,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/3680cd0506da4e5b92e8191e26439d57 is 50, key is test_row_0/B:col10/1731768802367/Put/seqid=0 2024-11-16T14:53:23,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742251_1427 (size=12001) 2024-11-16T14:53:23,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731768803594, max timestamp=1731768803619Num uniques 859; 2024-11-16T14:53:23,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731768803594, max timestamp=1731768803619Num uniques 859; 2024-11-16T14:53:23,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:53:23,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:53:23,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731768803594, max timestamp=1731768803619Num uniques 859; 2024-11-16T14:53:23,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T14:53:23,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768863684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768863690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T14:53:23,800 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:23,801 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-16T14:53:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:53:23,802 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:23,804 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:23,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:23,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768863800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:23,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768863811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:53:23,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768863910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:23,921 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768863930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:23,940 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:23,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T14:53:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:23,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:23,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/3680cd0506da4e5b92e8191e26439d57 2024-11-16T14:53:23,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/0a8502dcbb5d4bf4a78a67a7e7cf3eff is 50, key is test_row_0/C:col10/1731768802367/Put/seqid=0 2024-11-16T14:53:23,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742252_1428 (size=12001) 2024-11-16T14:53:24,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768864020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:24,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768864030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:24,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T14:53:24,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:24,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:53:24,132 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:53:24,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T14:53:24,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:24,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:24,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:24,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:24,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:24,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768864341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:24,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768864350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:24,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/0a8502dcbb5d4bf4a78a67a7e7cf3eff 2024-11-16T14:53:24,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/16b5099496c24c6387bcd3248fddb37b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/16b5099496c24c6387bcd3248fddb37b 2024-11-16T14:53:24,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/16b5099496c24c6387bcd3248fddb37b, entries=200, sequenceid=57, filesize=14.0 K 2024-11-16T14:53:24,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/3680cd0506da4e5b92e8191e26439d57 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3680cd0506da4e5b92e8191e26439d57 2024-11-16T14:53:24,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3680cd0506da4e5b92e8191e26439d57, entries=150, sequenceid=57, filesize=11.7 K 2024-11-16T14:53:24,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/0a8502dcbb5d4bf4a78a67a7e7cf3eff as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0a8502dcbb5d4bf4a78a67a7e7cf3eff 2024-11-16T14:53:24,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0a8502dcbb5d4bf4a78a67a7e7cf3eff, entries=150, sequenceid=57, filesize=11.7 K 2024-11-16T14:53:24,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d96afd6d0e8810c274529387dcd006b9 in 840ms, sequenceid=57, compaction requested=true 2024-11-16T14:53:24,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:24,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:24,388 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:24,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:24,388 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:24,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:24,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:24,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:24,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:24,388 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:24,388 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:24,389 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/B is initiating minor compaction (all files) 2024-11-16T14:53:24,389 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/A is initiating minor compaction (all files) 2024-11-16T14:53:24,389 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/B in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:24,389 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/A in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:24,389 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7bfbfadc4125499d8926a019ae27157b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/f8ceb3780db6409fbacefa80b0cb25f9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3680cd0506da4e5b92e8191e26439d57] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=35.2 K 2024-11-16T14:53:24,389 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/31aaef3bddc8415a97f6c5aedb58a261, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d05a6681007f4fb28ca6171e22cb5332, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/16b5099496c24c6387bcd3248fddb37b] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=37.4 K 2024-11-16T14:53:24,389 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bfbfadc4125499d8926a019ae27157b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731768801653 2024-11-16T14:53:24,389 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31aaef3bddc8415a97f6c5aedb58a261, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731768801653 2024-11-16T14:53:24,389 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d05a6681007f4fb28ca6171e22cb5332, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731768801726 2024-11-16T14:53:24,389 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f8ceb3780db6409fbacefa80b0cb25f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731768801726 2024-11-16T14:53:24,389 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16b5099496c24c6387bcd3248fddb37b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731768802367 2024-11-16T14:53:24,389 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3680cd0506da4e5b92e8191e26439d57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731768802367 2024-11-16T14:53:24,395 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#A#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:24,396 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#B#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:24,396 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/c0dd7d27558c437d832d246b1b1ccb52 is 50, key is test_row_0/A:col10/1731768802367/Put/seqid=0 2024-11-16T14:53:24,396 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/daa5d1fca8bb4c9399183c7425764f34 is 50, key is test_row_0/B:col10/1731768802367/Put/seqid=0 2024-11-16T14:53:24,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742254_1430 (size=12104) 2024-11-16T14:53:24,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742253_1429 (size=12104) 2024-11-16T14:53:24,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T14:53:24,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:24,414 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-16T14:53:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:24,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/c744fec316544385b36d9ac1fc9428b3 is 50, key is test_row_0/A:col10/1731768803682/Put/seqid=0 2024-11-16T14:53:24,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742255_1431 (size=12001) 2024-11-16T14:53:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:53:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:24,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:24,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768864623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:24,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768864740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:24,812 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/daa5d1fca8bb4c9399183c7425764f34 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/daa5d1fca8bb4c9399183c7425764f34 2024-11-16T14:53:24,812 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/c0dd7d27558c437d832d246b1b1ccb52 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c0dd7d27558c437d832d246b1b1ccb52 2024-11-16T14:53:24,815 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/B of d96afd6d0e8810c274529387dcd006b9 into daa5d1fca8bb4c9399183c7425764f34(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:24,815 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:24,815 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/B, priority=13, startTime=1731768804388; duration=0sec 2024-11-16T14:53:24,815 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/A of d96afd6d0e8810c274529387dcd006b9 into c0dd7d27558c437d832d246b1b1ccb52(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:24,815 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:24,815 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:B 2024-11-16T14:53:24,815 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:24,815 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/A, priority=13, startTime=1731768804388; duration=0sec 2024-11-16T14:53:24,815 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:24,815 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:24,815 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:A 2024-11-16T14:53:24,816 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:24,816 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/C is initiating minor compaction (all files) 2024-11-16T14:53:24,816 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/C in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:24,816 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/e403ae56e9f64d1eadd27eeef136d19c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d8c13af278cc4ed4a65c5cea8bd99342, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0a8502dcbb5d4bf4a78a67a7e7cf3eff] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=35.2 K 2024-11-16T14:53:24,816 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting e403ae56e9f64d1eadd27eeef136d19c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731768801653 2024-11-16T14:53:24,816 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c13af278cc4ed4a65c5cea8bd99342, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731768801726 2024-11-16T14:53:24,817 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a8502dcbb5d4bf4a78a67a7e7cf3eff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731768802367 2024-11-16T14:53:24,822 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#C#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:24,822 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/cbf87f5f4b444add96e750f5a0213f60 is 50, key is test_row_0/C:col10/1731768802367/Put/seqid=0 2024-11-16T14:53:24,823 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/c744fec316544385b36d9ac1fc9428b3 2024-11-16T14:53:24,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742256_1432 (size=12104) 2024-11-16T14:53:24,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/c7ab0b8539ec454eb312e8908985d2d2 is 50, key is test_row_0/B:col10/1731768803682/Put/seqid=0 2024-11-16T14:53:24,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742257_1433 (size=12001) 2024-11-16T14:53:24,834 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/cbf87f5f4b444add96e750f5a0213f60 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/cbf87f5f4b444add96e750f5a0213f60 2024-11-16T14:53:24,837 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/C of d96afd6d0e8810c274529387dcd006b9 into cbf87f5f4b444add96e750f5a0213f60(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:24,837 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:24,837 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/C, priority=13, startTime=1731768804388; duration=0sec 2024-11-16T14:53:24,838 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:24,838 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:C 2024-11-16T14:53:24,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768864860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:24,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768864860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:53:24,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768864950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:25,233 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/c7ab0b8539ec454eb312e8908985d2d2 2024-11-16T14:53:25,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/a2db803e965044bc9cfaab65fedb42df is 50, key is test_row_0/C:col10/1731768803682/Put/seqid=0 2024-11-16T14:53:25,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742258_1434 (size=12001) 2024-11-16T14:53:25,243 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/a2db803e965044bc9cfaab65fedb42df 2024-11-16T14:53:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/c744fec316544385b36d9ac1fc9428b3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c744fec316544385b36d9ac1fc9428b3 2024-11-16T14:53:25,249 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c744fec316544385b36d9ac1fc9428b3, entries=150, sequenceid=92, filesize=11.7 K 2024-11-16T14:53:25,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/c7ab0b8539ec454eb312e8908985d2d2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c7ab0b8539ec454eb312e8908985d2d2 2024-11-16T14:53:25,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c7ab0b8539ec454eb312e8908985d2d2, entries=150, sequenceid=92, filesize=11.7 K 2024-11-16T14:53:25,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/a2db803e965044bc9cfaab65fedb42df as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a2db803e965044bc9cfaab65fedb42df 2024-11-16T14:53:25,257 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a2db803e965044bc9cfaab65fedb42df, entries=150, sequenceid=92, filesize=11.7 K 2024-11-16T14:53:25,257 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for d96afd6d0e8810c274529387dcd006b9 in 843ms, sequenceid=92, compaction requested=false 2024-11-16T14:53:25,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-16T14:53:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-16T14:53:25,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-16T14:53:25,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4550 sec 2024-11-16T14:53:25,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4590 sec 2024-11-16T14:53:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:25,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:53:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:25,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/4ecfc9a714e34022b5e54c9cff7e1c21 is 50, key is test_row_0/A:col10/1731768804621/Put/seqid=0 2024-11-16T14:53:25,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742259_1435 (size=14341) 2024-11-16T14:53:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768865403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:25,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768865521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:25,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/4ecfc9a714e34022b5e54c9cff7e1c21 2024-11-16T14:53:25,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/389e4596c6dc49018efa7aeff1e89286 is 50, key is test_row_0/B:col10/1731768804621/Put/seqid=0 2024-11-16T14:53:25,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742260_1436 (size=12001) 2024-11-16T14:53:25,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768865740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:25,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768865871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:25,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768865871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:25,880 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:25,880 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:25,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768865940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:25,948 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:25,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:25,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T14:53:25,951 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:25,951 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-16T14:53:25,953 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:25,953 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:25,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T14:53:25,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768865961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:25,963 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T14:53:26,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768866061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:26,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/389e4596c6dc49018efa7aeff1e89286 2024-11-16T14:53:26,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T14:53:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:26,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:26,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:26,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/00705d7b2dba4ef2b33ea0b88d83f07c is 50, key is test_row_0/C:col10/1731768804621/Put/seqid=0 2024-11-16T14:53:26,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742261_1437 (size=12001) 2024-11-16T14:53:26,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T14:53:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:26,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:26,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:26,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T14:53:26,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T14:53:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:26,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:26,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/00705d7b2dba4ef2b33ea0b88d83f07c 2024-11-16T14:53:26,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/4ecfc9a714e34022b5e54c9cff7e1c21 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/4ecfc9a714e34022b5e54c9cff7e1c21 2024-11-16T14:53:26,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/4ecfc9a714e34022b5e54c9cff7e1c21, entries=200, sequenceid=107, filesize=14.0 K 2024-11-16T14:53:26,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/389e4596c6dc49018efa7aeff1e89286 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/389e4596c6dc49018efa7aeff1e89286 2024-11-16T14:53:26,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/389e4596c6dc49018efa7aeff1e89286, entries=150, sequenceid=107, filesize=11.7 K 2024-11-16T14:53:26,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/00705d7b2dba4ef2b33ea0b88d83f07c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/00705d7b2dba4ef2b33ea0b88d83f07c 2024-11-16T14:53:26,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/00705d7b2dba4ef2b33ea0b88d83f07c, entries=150, sequenceid=107, filesize=11.7 K 2024-11-16T14:53:26,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d96afd6d0e8810c274529387dcd006b9 in 1267ms, sequenceid=107, compaction requested=true 2024-11-16T14:53:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:26,541 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:26,541 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:26,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:26,542 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:26,542 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:26,542 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/A is initiating minor compaction (all files) 2024-11-16T14:53:26,542 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/B is initiating minor compaction (all files) 2024-11-16T14:53:26,542 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/A in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:26,542 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/B in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:26,542 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/daa5d1fca8bb4c9399183c7425764f34, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c7ab0b8539ec454eb312e8908985d2d2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/389e4596c6dc49018efa7aeff1e89286] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=35.3 K 2024-11-16T14:53:26,542 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c0dd7d27558c437d832d246b1b1ccb52, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c744fec316544385b36d9ac1fc9428b3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/4ecfc9a714e34022b5e54c9cff7e1c21] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=37.5 K 2024-11-16T14:53:26,543 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0dd7d27558c437d832d246b1b1ccb52, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731768802367 2024-11-16T14:53:26,543 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting daa5d1fca8bb4c9399183c7425764f34, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731768802367 2024-11-16T14:53:26,543 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting c7ab0b8539ec454eb312e8908985d2d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731768803619 2024-11-16T14:53:26,543 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c744fec316544385b36d9ac1fc9428b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731768803619 2024-11-16T14:53:26,543 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 389e4596c6dc49018efa7aeff1e89286, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731768804616 2024-11-16T14:53:26,543 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ecfc9a714e34022b5e54c9cff7e1c21, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731768804591 2024-11-16T14:53:26,550 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#B#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:26,550 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/273a08b8f5a64e64bfc9f6c574cacf36 is 50, key is test_row_0/B:col10/1731768804621/Put/seqid=0 2024-11-16T14:53:26,554 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#A#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:26,554 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/fe33c4019b1d480c8f54fd1c2215b218 is 50, key is test_row_0/A:col10/1731768804621/Put/seqid=0 2024-11-16T14:53:26,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T14:53:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:26,565 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:53:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:26,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742262_1438 (size=12207) 2024-11-16T14:53:26,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742263_1439 (size=12207) 2024-11-16T14:53:26,581 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/fe33c4019b1d480c8f54fd1c2215b218 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fe33c4019b1d480c8f54fd1c2215b218 2024-11-16T14:53:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T14:53:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/9c1ca5b666984cfebb0eefb99898ac0a is 50, key is test_row_0/A:col10/1731768805394/Put/seqid=0 2024-11-16T14:53:26,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:26,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742264_1440 (size=12001) 2024-11-16T14:53:26,589 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/A of d96afd6d0e8810c274529387dcd006b9 into fe33c4019b1d480c8f54fd1c2215b218(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:26,589 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:26,589 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/A, priority=13, startTime=1731768806541; duration=0sec 2024-11-16T14:53:26,589 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:26,589 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:A 2024-11-16T14:53:26,589 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:26,590 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/9c1ca5b666984cfebb0eefb99898ac0a 2024-11-16T14:53:26,593 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:26,593 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/C is initiating minor compaction (all files) 2024-11-16T14:53:26,593 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/C in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:26,593 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/cbf87f5f4b444add96e750f5a0213f60, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a2db803e965044bc9cfaab65fedb42df, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/00705d7b2dba4ef2b33ea0b88d83f07c] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=35.3 K 2024-11-16T14:53:26,593 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbf87f5f4b444add96e750f5a0213f60, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731768802367 2024-11-16T14:53:26,594 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2db803e965044bc9cfaab65fedb42df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731768803619 2024-11-16T14:53:26,594 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00705d7b2dba4ef2b33ea0b88d83f07c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731768804616 2024-11-16T14:53:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/4a35f70ec70141e4a6482ee12aa3dda5 is 50, key is test_row_0/B:col10/1731768805394/Put/seqid=0 2024-11-16T14:53:26,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742265_1441 (size=12001) 2024-11-16T14:53:26,617 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:26,617 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/aa311ef4532f4942931267f34a27bcf3 is 50, key is test_row_0/C:col10/1731768804621/Put/seqid=0 2024-11-16T14:53:26,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742266_1442 (size=12207) 2024-11-16T14:53:26,628 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/aa311ef4532f4942931267f34a27bcf3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/aa311ef4532f4942931267f34a27bcf3 2024-11-16T14:53:26,631 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/C of d96afd6d0e8810c274529387dcd006b9 into aa311ef4532f4942931267f34a27bcf3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:26,631 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:26,631 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/C, priority=13, startTime=1731768806541; duration=0sec 2024-11-16T14:53:26,631 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:26,632 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:C 2024-11-16T14:53:26,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768866653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:26,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768866771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:26,972 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/273a08b8f5a64e64bfc9f6c574cacf36 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/273a08b8f5a64e64bfc9f6c574cacf36 2024-11-16T14:53:26,975 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/B of d96afd6d0e8810c274529387dcd006b9 into 273a08b8f5a64e64bfc9f6c574cacf36(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:26,975 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:26,975 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/B, priority=13, startTime=1731768806541; duration=0sec 2024-11-16T14:53:26,975 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:26,975 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:B 2024-11-16T14:53:27,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768866991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:27,013 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/4a35f70ec70141e4a6482ee12aa3dda5 2024-11-16T14:53:27,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/9bfb912bc95e4a648e0dcb3bc5b33f7c is 50, key is test_row_0/C:col10/1731768805394/Put/seqid=0 2024-11-16T14:53:27,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742267_1443 (size=12001) 2024-11-16T14:53:27,037 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/9bfb912bc95e4a648e0dcb3bc5b33f7c 2024-11-16T14:53:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/9c1ca5b666984cfebb0eefb99898ac0a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/9c1ca5b666984cfebb0eefb99898ac0a 2024-11-16T14:53:27,044 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/9c1ca5b666984cfebb0eefb99898ac0a, entries=150, sequenceid=127, filesize=11.7 K 2024-11-16T14:53:27,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/4a35f70ec70141e4a6482ee12aa3dda5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4a35f70ec70141e4a6482ee12aa3dda5 2024-11-16T14:53:27,048 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4a35f70ec70141e4a6482ee12aa3dda5, entries=150, sequenceid=127, filesize=11.7 K 2024-11-16T14:53:27,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/9bfb912bc95e4a648e0dcb3bc5b33f7c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/9bfb912bc95e4a648e0dcb3bc5b33f7c 2024-11-16T14:53:27,053 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/9bfb912bc95e4a648e0dcb3bc5b33f7c, entries=150, sequenceid=127, filesize=11.7 K 2024-11-16T14:53:27,054 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d96afd6d0e8810c274529387dcd006b9 in 489ms, sequenceid=127, compaction requested=false 2024-11-16T14:53:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-16T14:53:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-16T14:53:27,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-16T14:53:27,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1010 sec 2024-11-16T14:53:27,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1060 sec 2024-11-16T14:53:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T14:53:27,091 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:27,091 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-16T14:53:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:53:27,093 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:27,094 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:27,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:53:27,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T14:53:27,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:27,246 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:53:27,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:27,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:27,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:27,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:27,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:27,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:27,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/8d159299e8284bb2a1512d553f439e9e is 50, key is test_row_0/A:col10/1731768806634/Put/seqid=0 2024-11-16T14:53:27,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742268_1444 (size=12151) 2024-11-16T14:53:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:27,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:53:27,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768867455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:27,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768867570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:27,655 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/8d159299e8284bb2a1512d553f439e9e 2024-11-16T14:53:27,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/6cd1ee21e0344e84987ee39fd83058e1 is 50, key is test_row_0/B:col10/1731768806634/Put/seqid=0 2024-11-16T14:53:27,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742269_1445 (size=12151) 2024-11-16T14:53:27,668 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/6cd1ee21e0344e84987ee39fd83058e1 2024-11-16T14:53:27,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/f0db9c4e0a834815b3c64bc11598a220 is 50, key is test_row_0/C:col10/1731768806634/Put/seqid=0 2024-11-16T14:53:27,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742270_1446 (size=12151) 2024-11-16T14:53:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:53:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768867791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:27,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768867891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:27,898 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:27,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:27,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768867900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:27,910 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:27,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:27,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:28,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768868100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:28,117 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/f0db9c4e0a834815b3c64bc11598a220 2024-11-16T14:53:28,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/8d159299e8284bb2a1512d553f439e9e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/8d159299e8284bb2a1512d553f439e9e 2024-11-16T14:53:28,122 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/8d159299e8284bb2a1512d553f439e9e, entries=150, sequenceid=142, filesize=11.9 K 2024-11-16T14:53:28,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/6cd1ee21e0344e84987ee39fd83058e1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/6cd1ee21e0344e84987ee39fd83058e1 2024-11-16T14:53:28,125 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/6cd1ee21e0344e84987ee39fd83058e1, entries=150, sequenceid=142, filesize=11.9 K 2024-11-16T14:53:28,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/f0db9c4e0a834815b3c64bc11598a220 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f0db9c4e0a834815b3c64bc11598a220 2024-11-16T14:53:28,128 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f0db9c4e0a834815b3c64bc11598a220, entries=150, sequenceid=142, filesize=11.9 K 2024-11-16T14:53:28,129 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d96afd6d0e8810c274529387dcd006b9 in 883ms, sequenceid=142, compaction requested=true 2024-11-16T14:53:28,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:28,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:28,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-16T14:53:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-16T14:53:28,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-16T14:53:28,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0360 sec 2024-11-16T14:53:28,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.0410 sec 2024-11-16T14:53:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T14:53:28,231 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:28,231 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-16T14:53:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:53:28,233 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:28,233 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:28,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:53:28,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T14:53:28,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:28,385 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:53:28,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:28,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:28,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:28,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:28,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:28,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:28,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/32bc76295a6849adb00f9c4856ac0dce is 50, key is test_row_0/A:col10/1731768807406/Put/seqid=0 2024-11-16T14:53:28,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742271_1447 (size=12151) 2024-11-16T14:53:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:53:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:28,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768868704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:28,794 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/32bc76295a6849adb00f9c4856ac0dce 2024-11-16T14:53:28,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/ce6cc8c0e5284644a02733cb04acfa33 is 50, key is test_row_0/B:col10/1731768807406/Put/seqid=0 2024-11-16T14:53:28,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742272_1448 (size=12151) 2024-11-16T14:53:28,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768868820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:53:29,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768869040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:29,204 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/ce6cc8c0e5284644a02733cb04acfa33 2024-11-16T14:53:29,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/2d7110cb2c9b4cab9b6507904a7121d4 is 50, key is test_row_0/C:col10/1731768807406/Put/seqid=0 2024-11-16T14:53:29,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742273_1449 (size=12151) 2024-11-16T14:53:29,214 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/2d7110cb2c9b4cab9b6507904a7121d4 2024-11-16T14:53:29,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/32bc76295a6849adb00f9c4856ac0dce as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/32bc76295a6849adb00f9c4856ac0dce 2024-11-16T14:53:29,220 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/32bc76295a6849adb00f9c4856ac0dce, entries=150, sequenceid=162, filesize=11.9 K 2024-11-16T14:53:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/ce6cc8c0e5284644a02733cb04acfa33 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ce6cc8c0e5284644a02733cb04acfa33 2024-11-16T14:53:29,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ce6cc8c0e5284644a02733cb04acfa33, entries=150, sequenceid=162, filesize=11.9 K 2024-11-16T14:53:29,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/2d7110cb2c9b4cab9b6507904a7121d4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2d7110cb2c9b4cab9b6507904a7121d4 2024-11-16T14:53:29,226 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2d7110cb2c9b4cab9b6507904a7121d4, entries=150, sequenceid=162, filesize=11.9 K 2024-11-16T14:53:29,230 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d96afd6d0e8810c274529387dcd006b9 in 845ms, sequenceid=162, compaction requested=true 2024-11-16T14:53:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:29,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-16T14:53:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-16T14:53:29,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-16T14:53:29,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 998 msec 2024-11-16T14:53:29,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0020 sec 2024-11-16T14:53:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:29,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:53:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:29,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:29,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/f54595ccbfc448c2ad8c70eea1065392 is 50, key is test_row_0/A:col10/1731768808684/Put/seqid=0 2024-11-16T14:53:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T14:53:29,371 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:29,371 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-16T14:53:29,374 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:53:29,375 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:29,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:29,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742274_1450 (size=14541) 2024-11-16T14:53:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:53:29,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:29,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:29,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:29,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:29,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768869522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:29,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768869650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:29,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:29,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:29,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:53:29,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/f54595ccbfc448c2ad8c70eea1065392 2024-11-16T14:53:29,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/a563dbf8154b4b94b6369a683a006e09 is 50, key is test_row_0/B:col10/1731768808684/Put/seqid=0 2024-11-16T14:53:29,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742275_1451 (size=12151) 2024-11-16T14:53:29,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:29,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:29,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:29,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:29,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768869861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:29,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768869961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:29,966 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:29,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:29,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:29,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768869981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:29,993 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:53:30,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768870180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:30,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/a563dbf8154b4b94b6369a683a006e09 2024-11-16T14:53:30,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/3e74b80106e9464486fff06db0b0fd2e is 50, key is test_row_0/C:col10/1731768808684/Put/seqid=0 2024-11-16T14:53:30,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742276_1452 (size=12151) 2024-11-16T14:53:30,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:30,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:30,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:30,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:30,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:53:30,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:30,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/3e74b80106e9464486fff06db0b0fd2e 2024-11-16T14:53:30,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/f54595ccbfc448c2ad8c70eea1065392 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/f54595ccbfc448c2ad8c70eea1065392 2024-11-16T14:53:30,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/f54595ccbfc448c2ad8c70eea1065392, entries=200, sequenceid=175, filesize=14.2 K 2024-11-16T14:53:30,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/a563dbf8154b4b94b6369a683a006e09 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/a563dbf8154b4b94b6369a683a006e09 2024-11-16T14:53:30,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/a563dbf8154b4b94b6369a683a006e09, entries=150, sequenceid=175, filesize=11.9 K 2024-11-16T14:53:30,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/3e74b80106e9464486fff06db0b0fd2e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e74b80106e9464486fff06db0b0fd2e 2024-11-16T14:53:30,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e74b80106e9464486fff06db0b0fd2e, entries=150, sequenceid=175, filesize=11.9 K 2024-11-16T14:53:30,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d96afd6d0e8810c274529387dcd006b9 in 1267ms, sequenceid=175, compaction requested=true 2024-11-16T14:53:30,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:30,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:30,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:30,624 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T14:53:30,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:30,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:30,625 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T14:53:30,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:30,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:30,625 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63051 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T14:53:30,625 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T14:53:30,626 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/B is initiating minor compaction (all files) 2024-11-16T14:53:30,626 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/A is initiating minor compaction (all files) 2024-11-16T14:53:30,626 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/A in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,626 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/B in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,626 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/273a08b8f5a64e64bfc9f6c574cacf36, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4a35f70ec70141e4a6482ee12aa3dda5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/6cd1ee21e0344e84987ee39fd83058e1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ce6cc8c0e5284644a02733cb04acfa33, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/a563dbf8154b4b94b6369a683a006e09] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=59.2 K 2024-11-16T14:53:30,626 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fe33c4019b1d480c8f54fd1c2215b218, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/9c1ca5b666984cfebb0eefb99898ac0a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/8d159299e8284bb2a1512d553f439e9e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/32bc76295a6849adb00f9c4856ac0dce, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/f54595ccbfc448c2ad8c70eea1065392] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=61.6 K 2024-11-16T14:53:30,626 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 273a08b8f5a64e64bfc9f6c574cacf36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731768804616 2024-11-16T14:53:30,626 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe33c4019b1d480c8f54fd1c2215b218, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731768804616 2024-11-16T14:53:30,626 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a35f70ec70141e4a6482ee12aa3dda5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731768805355 2024-11-16T14:53:30,626 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c1ca5b666984cfebb0eefb99898ac0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731768805355 2024-11-16T14:53:30,627 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d159299e8284bb2a1512d553f439e9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731768806632 2024-11-16T14:53:30,627 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cd1ee21e0344e84987ee39fd83058e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731768806632 2024-11-16T14:53:30,627 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32bc76295a6849adb00f9c4856ac0dce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731768807406 2024-11-16T14:53:30,627 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ce6cc8c0e5284644a02733cb04acfa33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731768807406 2024-11-16T14:53:30,627 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f54595ccbfc448c2ad8c70eea1065392, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731768808652 2024-11-16T14:53:30,627 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting a563dbf8154b4b94b6369a683a006e09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731768808684 2024-11-16T14:53:30,636 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:30,636 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#B#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:30,636 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/abdc529c71394fc3be42c69e64f4b739 is 50, key is test_row_0/B:col10/1731768808684/Put/seqid=0 2024-11-16T14:53:30,636 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/363db5277f66456c928c2feb8b5d8416 is 50, key is test_row_0/A:col10/1731768808684/Put/seqid=0 2024-11-16T14:53:30,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742278_1454 (size=12527) 2024-11-16T14:53:30,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742277_1453 (size=12527) 2024-11-16T14:53:30,651 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/363db5277f66456c928c2feb8b5d8416 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/363db5277f66456c928c2feb8b5d8416 2024-11-16T14:53:30,654 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/A of d96afd6d0e8810c274529387dcd006b9 into 363db5277f66456c928c2feb8b5d8416(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:30,654 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:30,654 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/A, priority=11, startTime=1731768810624; duration=0sec 2024-11-16T14:53:30,655 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:30,655 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:A 2024-11-16T14:53:30,655 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T14:53:30,656 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T14:53:30,656 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/C is initiating minor compaction (all files) 2024-11-16T14:53:30,656 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/C in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,656 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/aa311ef4532f4942931267f34a27bcf3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/9bfb912bc95e4a648e0dcb3bc5b33f7c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f0db9c4e0a834815b3c64bc11598a220, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2d7110cb2c9b4cab9b6507904a7121d4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e74b80106e9464486fff06db0b0fd2e] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=59.2 K 2024-11-16T14:53:30,656 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa311ef4532f4942931267f34a27bcf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731768804616 2024-11-16T14:53:30,657 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bfb912bc95e4a648e0dcb3bc5b33f7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731768805355 2024-11-16T14:53:30,657 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0db9c4e0a834815b3c64bc11598a220, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731768806632 2024-11-16T14:53:30,657 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d7110cb2c9b4cab9b6507904a7121d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731768807406 2024-11-16T14:53:30,657 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e74b80106e9464486fff06db0b0fd2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731768808684 2024-11-16T14:53:30,668 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#C#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:30,668 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/f29c4e91b59449958c84bb6df752421b is 50, key is test_row_0/C:col10/1731768808684/Put/seqid=0 2024-11-16T14:53:30,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742279_1455 (size=12527) 2024-11-16T14:53:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:30,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T14:53:30,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:30,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:30,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:30,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:30,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:30,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:30,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/0a9bdb670bc9465298b5ed50a49582c4 is 50, key is test_row_0/A:col10/1731768809518/Put/seqid=0 2024-11-16T14:53:30,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742280_1456 (size=14541) 2024-11-16T14:53:30,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:30,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:30,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768870787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768870900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:30,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:30,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,043 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/abdc529c71394fc3be42c69e64f4b739 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/abdc529c71394fc3be42c69e64f4b739 2024-11-16T14:53:31,047 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/B of d96afd6d0e8810c274529387dcd006b9 into abdc529c71394fc3be42c69e64f4b739(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:31,047 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:31,047 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/B, priority=11, startTime=1731768810624; duration=0sec 2024-11-16T14:53:31,047 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:31,047 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:B 2024-11-16T14:53:31,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:31,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:31,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,077 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/f29c4e91b59449958c84bb6df752421b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f29c4e91b59449958c84bb6df752421b 2024-11-16T14:53:31,081 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/C of d96afd6d0e8810c274529387dcd006b9 into f29c4e91b59449958c84bb6df752421b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:31,081 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:31,081 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/C, priority=11, startTime=1731768810625; duration=0sec 2024-11-16T14:53:31,081 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:31,081 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:C 2024-11-16T14:53:31,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768871110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:31,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:31,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/0a9bdb670bc9465298b5ed50a49582c4 2024-11-16T14:53:31,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/7430ed0687044506a5505fd1d710d3f7 is 50, key is test_row_0/B:col10/1731768809518/Put/seqid=0 2024-11-16T14:53:31,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742281_1457 (size=12151) 2024-11-16T14:53:31,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:31,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:31,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768871420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:53:31,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/7430ed0687044506a5505fd1d710d3f7 2024-11-16T14:53:31,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/7aa41ca4339d4a0dadae223202f4a700 is 50, key is test_row_0/C:col10/1731768809518/Put/seqid=0 2024-11-16T14:53:31,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742282_1458 (size=12151) 2024-11-16T14:53:31,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:31,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768871920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:31,926 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:31,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:31,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768871930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:31,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:31,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:31,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:31,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:31,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768871941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:31,947 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:31,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/7aa41ca4339d4a0dadae223202f4a700 2024-11-16T14:53:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/0a9bdb670bc9465298b5ed50a49582c4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0a9bdb670bc9465298b5ed50a49582c4 2024-11-16T14:53:31,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0a9bdb670bc9465298b5ed50a49582c4, entries=200, sequenceid=196, filesize=14.2 K 2024-11-16T14:53:31,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/7430ed0687044506a5505fd1d710d3f7 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7430ed0687044506a5505fd1d710d3f7 2024-11-16T14:53:31,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7430ed0687044506a5505fd1d710d3f7, entries=150, sequenceid=196, filesize=11.9 K 2024-11-16T14:53:31,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/7aa41ca4339d4a0dadae223202f4a700 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/7aa41ca4339d4a0dadae223202f4a700 2024-11-16T14:53:31,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/7aa41ca4339d4a0dadae223202f4a700, entries=150, sequenceid=196, filesize=11.9 K 2024-11-16T14:53:31,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d96afd6d0e8810c274529387dcd006b9 in 1270ms, sequenceid=196, compaction requested=false 2024-11-16T14:53:31,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:31,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T14:53:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:31,980 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:53:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:31,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/5989294762d64cc18678e59e38621955 is 50, key is test_row_0/A:col10/1731768810760/Put/seqid=0 2024-11-16T14:53:31,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742283_1459 (size=12151) 2024-11-16T14:53:32,397 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/5989294762d64cc18678e59e38621955 2024-11-16T14:53:32,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/4081d05bd99e48ae87b126b9c5d60494 is 50, key is test_row_0/B:col10/1731768810760/Put/seqid=0 2024-11-16T14:53:32,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742284_1460 (size=12151) 2024-11-16T14:53:32,463 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/4081d05bd99e48ae87b126b9c5d60494 2024-11-16T14:53:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/a4a6c3b7c3824b5bb17c6fae6ae7bb82 is 50, key is test_row_0/C:col10/1731768810760/Put/seqid=0 2024-11-16T14:53:32,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742285_1461 (size=12151) 2024-11-16T14:53:32,514 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/a4a6c3b7c3824b5bb17c6fae6ae7bb82 2024-11-16T14:53:32,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/5989294762d64cc18678e59e38621955 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5989294762d64cc18678e59e38621955 2024-11-16T14:53:32,532 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5989294762d64cc18678e59e38621955, entries=150, sequenceid=210, filesize=11.9 K 2024-11-16T14:53:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/4081d05bd99e48ae87b126b9c5d60494 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4081d05bd99e48ae87b126b9c5d60494 2024-11-16T14:53:32,540 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4081d05bd99e48ae87b126b9c5d60494, entries=150, sequenceid=210, filesize=11.9 K 2024-11-16T14:53:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/a4a6c3b7c3824b5bb17c6fae6ae7bb82 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a4a6c3b7c3824b5bb17c6fae6ae7bb82 2024-11-16T14:53:32,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a4a6c3b7c3824b5bb17c6fae6ae7bb82, entries=150, sequenceid=210, filesize=11.9 K 2024-11-16T14:53:32,547 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for d96afd6d0e8810c274529387dcd006b9 in 567ms, sequenceid=210, compaction requested=true 2024-11-16T14:53:32,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:32,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:32,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-16T14:53:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-16T14:53:32,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-16T14:53:32,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1740 sec 2024-11-16T14:53:32,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.1800 sec 2024-11-16T14:53:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:32,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:53:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:32,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/a2a484f136eb471f944fdae976e82921 is 50, key is test_row_0/A:col10/1731768812982/Put/seqid=0 2024-11-16T14:53:32,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742286_1462 (size=16931) 2024-11-16T14:53:33,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768813087, max timestamp=1731768813140Num uniques 859; 2024-11-16T14:53:33,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768813087, max timestamp=1731768813140Num uniques 859; 2024-11-16T14:53:33,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:33,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:33,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768813087, max timestamp=1731768813140Num uniques 859; 2024-11-16T14:53:33,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768873291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:33,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/a2a484f136eb471f944fdae976e82921 2024-11-16T14:53:33,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/c07bc75bc0414542b7c9756a8af2a703 is 50, key is test_row_0/B:col10/1731768812982/Put/seqid=0 2024-11-16T14:53:33,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742287_1463 (size=12151) 2024-11-16T14:53:33,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768873404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T14:53:33,541 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:33,541 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-16T14:53:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:53:33,543 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:33,543 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:33,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:33,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768873630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:53:33,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T14:53:33,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:33,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:33,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:33,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:33,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:33,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/c07bc75bc0414542b7c9756a8af2a703 2024-11-16T14:53:33,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/0ee260eaa8ef4f61ae6d319effac83c5 is 50, key is test_row_0/C:col10/1731768812982/Put/seqid=0 2024-11-16T14:53:33,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742288_1464 (size=12151) 2024-11-16T14:53:33,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/0ee260eaa8ef4f61ae6d319effac83c5 2024-11-16T14:53:33,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/a2a484f136eb471f944fdae976e82921 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/a2a484f136eb471f944fdae976e82921 2024-11-16T14:53:33,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/a2a484f136eb471f944fdae976e82921, entries=250, sequenceid=220, filesize=16.5 K 2024-11-16T14:53:33,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/c07bc75bc0414542b7c9756a8af2a703 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c07bc75bc0414542b7c9756a8af2a703 2024-11-16T14:53:33,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c07bc75bc0414542b7c9756a8af2a703, entries=150, sequenceid=220, filesize=11.9 K 2024-11-16T14:53:33,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/0ee260eaa8ef4f61ae6d319effac83c5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0ee260eaa8ef4f61ae6d319effac83c5 2024-11-16T14:53:33,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0ee260eaa8ef4f61ae6d319effac83c5, entries=150, sequenceid=220, filesize=11.9 K 2024-11-16T14:53:33,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d96afd6d0e8810c274529387dcd006b9 in 853ms, sequenceid=220, compaction requested=true 2024-11-16T14:53:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:33,840 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:53:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:33,840 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:53:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:33,841 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56150 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:53:33,841 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:53:33,841 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/B is initiating minor compaction (all files) 2024-11-16T14:53:33,841 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/A is initiating minor compaction (all files) 2024-11-16T14:53:33,841 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/A in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:33,841 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/B in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:33,841 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/abdc529c71394fc3be42c69e64f4b739, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7430ed0687044506a5505fd1d710d3f7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4081d05bd99e48ae87b126b9c5d60494, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c07bc75bc0414542b7c9756a8af2a703] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=47.8 K 2024-11-16T14:53:33,841 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/363db5277f66456c928c2feb8b5d8416, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0a9bdb670bc9465298b5ed50a49582c4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5989294762d64cc18678e59e38621955, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/a2a484f136eb471f944fdae976e82921] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=54.8 K 2024-11-16T14:53:33,841 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting abdc529c71394fc3be42c69e64f4b739, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731768808684 2024-11-16T14:53:33,841 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 363db5277f66456c928c2feb8b5d8416, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731768808684 2024-11-16T14:53:33,841 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9bdb670bc9465298b5ed50a49582c4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731768809477 2024-11-16T14:53:33,841 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7430ed0687044506a5505fd1d710d3f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731768809477 2024-11-16T14:53:33,841 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5989294762d64cc18678e59e38621955, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731768810760 2024-11-16T14:53:33,841 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4081d05bd99e48ae87b126b9c5d60494, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731768810760 2024-11-16T14:53:33,842 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2a484f136eb471f944fdae976e82921, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731768812960 2024-11-16T14:53:33,842 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting c07bc75bc0414542b7c9756a8af2a703, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731768812979 2024-11-16T14:53:33,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T14:53:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:33,848 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T14:53:33,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:33,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:33,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:33,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:33,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:33,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:33,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/833ae86b74ad492faaf0aec9ebf059fc is 50, key is test_row_0/A:col10/1731768813289/Put/seqid=0 2024-11-16T14:53:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:53:33,863 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:33,863 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#A#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:33,864 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/d601a722af684861a9103307537360e3 is 50, key is test_row_0/B:col10/1731768812982/Put/seqid=0 2024-11-16T14:53:33,864 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/7556fe34d70f4eef8d8abae24f535cd2 is 50, key is test_row_0/A:col10/1731768812982/Put/seqid=0 2024-11-16T14:53:33,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742289_1465 (size=12151) 2024-11-16T14:53:33,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742290_1466 (size=12663) 2024-11-16T14:53:33,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742291_1467 (size=12663) 2024-11-16T14:53:33,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:34,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768874017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:34,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768874131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:53:34,266 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/833ae86b74ad492faaf0aec9ebf059fc 2024-11-16T14:53:34,272 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/d601a722af684861a9103307537360e3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/d601a722af684861a9103307537360e3 2024-11-16T14:53:34,276 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/7556fe34d70f4eef8d8abae24f535cd2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/7556fe34d70f4eef8d8abae24f535cd2 2024-11-16T14:53:34,277 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/B of d96afd6d0e8810c274529387dcd006b9 into d601a722af684861a9103307537360e3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:34,277 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:34,277 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/B, priority=12, startTime=1731768813840; duration=0sec 2024-11-16T14:53:34,277 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:34,277 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:B 2024-11-16T14:53:34,277 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:53:34,278 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:53:34,278 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/C is initiating minor compaction (all files) 2024-11-16T14:53:34,278 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/C in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:34,278 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f29c4e91b59449958c84bb6df752421b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/7aa41ca4339d4a0dadae223202f4a700, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a4a6c3b7c3824b5bb17c6fae6ae7bb82, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0ee260eaa8ef4f61ae6d319effac83c5] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=47.8 K 2024-11-16T14:53:34,278 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f29c4e91b59449958c84bb6df752421b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731768808684 2024-11-16T14:53:34,278 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa41ca4339d4a0dadae223202f4a700, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731768809477 2024-11-16T14:53:34,278 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a6c3b7c3824b5bb17c6fae6ae7bb82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731768810760 2024-11-16T14:53:34,279 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ee260eaa8ef4f61ae6d319effac83c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731768812979 2024-11-16T14:53:34,280 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/A of d96afd6d0e8810c274529387dcd006b9 into 7556fe34d70f4eef8d8abae24f535cd2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:34,280 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:34,280 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/A, priority=12, startTime=1731768813840; duration=0sec 2024-11-16T14:53:34,280 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:34,280 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:A 2024-11-16T14:53:34,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/c3d55c5666d34784b8af90f1081c29c1 is 50, key is test_row_0/B:col10/1731768813289/Put/seqid=0 2024-11-16T14:53:34,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742292_1468 (size=12151) 2024-11-16T14:53:34,292 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#C#compaction#397 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:34,292 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/ef7bbd0dd34746758d44a69ee0318156 is 50, key is test_row_0/C:col10/1731768812982/Put/seqid=0 2024-11-16T14:53:34,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742293_1469 (size=12663) 2024-11-16T14:53:34,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768874350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:34,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:34,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:34,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:53:34,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768874670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:34,695 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/c3d55c5666d34784b8af90f1081c29c1 2024-11-16T14:53:34,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/609157fdaea44656971b9c829b85e5da is 50, key is test_row_0/C:col10/1731768813289/Put/seqid=0 2024-11-16T14:53:34,713 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/ef7bbd0dd34746758d44a69ee0318156 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/ef7bbd0dd34746758d44a69ee0318156 2024-11-16T14:53:34,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742294_1470 (size=12151) 2024-11-16T14:53:34,719 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/C of d96afd6d0e8810c274529387dcd006b9 into ef7bbd0dd34746758d44a69ee0318156(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:34,719 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/609157fdaea44656971b9c829b85e5da 2024-11-16T14:53:34,719 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:34,719 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/C, priority=12, startTime=1731768813840; duration=0sec 2024-11-16T14:53:34,719 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:34,719 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:C 2024-11-16T14:53:34,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/833ae86b74ad492faaf0aec9ebf059fc as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/833ae86b74ad492faaf0aec9ebf059fc 2024-11-16T14:53:34,730 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/833ae86b74ad492faaf0aec9ebf059fc, entries=150, sequenceid=257, filesize=11.9 K 2024-11-16T14:53:34,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/c3d55c5666d34784b8af90f1081c29c1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c3d55c5666d34784b8af90f1081c29c1 2024-11-16T14:53:34,736 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c3d55c5666d34784b8af90f1081c29c1, entries=150, sequenceid=257, filesize=11.9 K 2024-11-16T14:53:34,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/609157fdaea44656971b9c829b85e5da as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/609157fdaea44656971b9c829b85e5da 2024-11-16T14:53:34,740 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/609157fdaea44656971b9c829b85e5da, entries=150, sequenceid=257, filesize=11.9 K 2024-11-16T14:53:34,741 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for d96afd6d0e8810c274529387dcd006b9 in 893ms, sequenceid=257, compaction requested=false 2024-11-16T14:53:34,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:34,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:34,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-16T14:53:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-16T14:53:34,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-16T14:53:34,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1980 sec 2024-11-16T14:53:34,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.2070 sec 2024-11-16T14:53:35,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:53:35,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:35,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:35,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:35,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:35,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:35,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:35,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/dd703dc7d8e74ff789e8aaa523d2dabb is 50, key is test_row_0/A:col10/1731768815193/Put/seqid=0 2024-11-16T14:53:35,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742295_1471 (size=14741) 2024-11-16T14:53:35,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/dd703dc7d8e74ff789e8aaa523d2dabb 2024-11-16T14:53:35,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/055bf1b27fb044c5a6b846b554b0cb43 is 50, key is test_row_0/B:col10/1731768815193/Put/seqid=0 2024-11-16T14:53:35,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742296_1472 (size=12301) 2024-11-16T14:53:35,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/055bf1b27fb044c5a6b846b554b0cb43 2024-11-16T14:53:35,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/d623d0bd19614f3783d1809b72766423 is 50, key is test_row_0/C:col10/1731768815193/Put/seqid=0 2024-11-16T14:53:35,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742297_1473 (size=12301) 2024-11-16T14:53:35,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768815310, max timestamp=1731768815392Num uniques 859; 2024-11-16T14:53:35,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768815310, max timestamp=1731768815392Num uniques 859; 2024-11-16T14:53:35,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:35,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768815310, max timestamp=1731768815392Num uniques 859; 2024-11-16T14:53:35,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:35,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:35,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768875595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T14:53:35,691 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:35,692 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-16T14:53:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T14:53:35,699 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:35,700 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:35,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:35,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/d623d0bd19614f3783d1809b72766423 2024-11-16T14:53:35,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/dd703dc7d8e74ff789e8aaa523d2dabb as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/dd703dc7d8e74ff789e8aaa523d2dabb 2024-11-16T14:53:35,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/dd703dc7d8e74ff789e8aaa523d2dabb, entries=200, sequenceid=271, filesize=14.4 K 2024-11-16T14:53:35,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/055bf1b27fb044c5a6b846b554b0cb43 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/055bf1b27fb044c5a6b846b554b0cb43 2024-11-16T14:53:35,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/055bf1b27fb044c5a6b846b554b0cb43, entries=150, sequenceid=271, filesize=12.0 K 2024-11-16T14:53:35,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/d623d0bd19614f3783d1809b72766423 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d623d0bd19614f3783d1809b72766423 2024-11-16T14:53:35,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768875701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:35,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d623d0bd19614f3783d1809b72766423, entries=150, sequenceid=271, filesize=12.0 K 2024-11-16T14:53:35,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d96afd6d0e8810c274529387dcd006b9 in 532ms, sequenceid=271, compaction requested=true 2024-11-16T14:53:35,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:35,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:35,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:35,727 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:35,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:35,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:35,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:35,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T14:53:35,729 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:35,731 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:35,731 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/A is initiating minor compaction (all files) 2024-11-16T14:53:35,731 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/A in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:35,731 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/7556fe34d70f4eef8d8abae24f535cd2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/833ae86b74ad492faaf0aec9ebf059fc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/dd703dc7d8e74ff789e8aaa523d2dabb] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=38.6 K 2024-11-16T14:53:35,732 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:35,732 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/B is initiating minor compaction (all files) 2024-11-16T14:53:35,732 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/B in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:35,732 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/d601a722af684861a9103307537360e3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c3d55c5666d34784b8af90f1081c29c1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/055bf1b27fb044c5a6b846b554b0cb43] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=36.2 K 2024-11-16T14:53:35,732 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 7556fe34d70f4eef8d8abae24f535cd2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731768812979 2024-11-16T14:53:35,732 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d601a722af684861a9103307537360e3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731768812979 2024-11-16T14:53:35,732 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 833ae86b74ad492faaf0aec9ebf059fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731768813176 2024-11-16T14:53:35,733 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting dd703dc7d8e74ff789e8aaa523d2dabb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731768813986 2024-11-16T14:53:35,733 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3d55c5666d34784b8af90f1081c29c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731768813176 2024-11-16T14:53:35,733 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 055bf1b27fb044c5a6b846b554b0cb43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731768813986 2024-11-16T14:53:35,740 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:35,740 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/ecf5daf3dc51416ea01950827a979a10 is 50, key is test_row_0/B:col10/1731768815193/Put/seqid=0 2024-11-16T14:53:35,745 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#A#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:35,745 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/3ce8ef20b096437caf0eb6dc4d153aae is 50, key is test_row_0/A:col10/1731768815193/Put/seqid=0 2024-11-16T14:53:35,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742298_1474 (size=12915) 2024-11-16T14:53:35,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742299_1475 (size=12915) 2024-11-16T14:53:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T14:53:35,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T14:53:35,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:35,853 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T14:53:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:35,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/fd965ee308dd4f7dbb0cad22c961dac5 is 50, key is test_row_0/A:col10/1731768815590/Put/seqid=0 2024-11-16T14:53:35,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742300_1476 (size=12301) 2024-11-16T14:53:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:35,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T14:53:36,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768876047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:36,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768876161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:36,193 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/ecf5daf3dc51416ea01950827a979a10 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ecf5daf3dc51416ea01950827a979a10 2024-11-16T14:53:36,197 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/B of d96afd6d0e8810c274529387dcd006b9 into ecf5daf3dc51416ea01950827a979a10(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:36,197 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:36,197 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/B, priority=13, startTime=1731768815727; duration=0sec 2024-11-16T14:53:36,198 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:36,198 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:B 2024-11-16T14:53:36,198 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:36,198 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:36,199 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/C is initiating minor compaction (all files) 2024-11-16T14:53:36,199 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/C in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:36,199 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/ef7bbd0dd34746758d44a69ee0318156, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/609157fdaea44656971b9c829b85e5da, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d623d0bd19614f3783d1809b72766423] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=36.2 K 2024-11-16T14:53:36,199 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef7bbd0dd34746758d44a69ee0318156, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731768812979 2024-11-16T14:53:36,206 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 609157fdaea44656971b9c829b85e5da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731768813176 2024-11-16T14:53:36,206 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d623d0bd19614f3783d1809b72766423, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731768813986 2024-11-16T14:53:36,207 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/3ce8ef20b096437caf0eb6dc4d153aae as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3ce8ef20b096437caf0eb6dc4d153aae 2024-11-16T14:53:36,212 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/A of d96afd6d0e8810c274529387dcd006b9 into 3ce8ef20b096437caf0eb6dc4d153aae(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:36,212 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:36,212 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/A, priority=13, startTime=1731768815727; duration=0sec 2024-11-16T14:53:36,212 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:36,212 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:A 2024-11-16T14:53:36,214 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:36,215 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/bd72aac844844057bf02a10c27689275 is 50, key is test_row_0/C:col10/1731768815193/Put/seqid=0 2024-11-16T14:53:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742301_1477 (size=12915) 2024-11-16T14:53:36,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/fd965ee308dd4f7dbb0cad22c961dac5 2024-11-16T14:53:36,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/b795ea1979044af5b741cab6410e2210 is 50, key is test_row_0/B:col10/1731768815590/Put/seqid=0 2024-11-16T14:53:36,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742302_1478 (size=12301) 2024-11-16T14:53:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T14:53:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768876370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:36,659 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/bd72aac844844057bf02a10c27689275 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/bd72aac844844057bf02a10c27689275 2024-11-16T14:53:36,663 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/C of d96afd6d0e8810c274529387dcd006b9 into bd72aac844844057bf02a10c27689275(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:36,663 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:36,663 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/C, priority=13, startTime=1731768815727; duration=0sec 2024-11-16T14:53:36,663 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:36,663 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:C 2024-11-16T14:53:36,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768876691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:36,728 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/b795ea1979044af5b741cab6410e2210 2024-11-16T14:53:36,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/66dd2631d6674a9caa8d5fe8e94a7dac is 50, key is test_row_0/C:col10/1731768815590/Put/seqid=0 2024-11-16T14:53:36,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742303_1479 (size=12301) 2024-11-16T14:53:36,750 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/66dd2631d6674a9caa8d5fe8e94a7dac 2024-11-16T14:53:36,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/fd965ee308dd4f7dbb0cad22c961dac5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fd965ee308dd4f7dbb0cad22c961dac5 2024-11-16T14:53:36,764 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fd965ee308dd4f7dbb0cad22c961dac5, entries=150, sequenceid=307, filesize=12.0 K 2024-11-16T14:53:36,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/b795ea1979044af5b741cab6410e2210 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b795ea1979044af5b741cab6410e2210 2024-11-16T14:53:36,778 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b795ea1979044af5b741cab6410e2210, entries=150, sequenceid=307, filesize=12.0 K 2024-11-16T14:53:36,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/66dd2631d6674a9caa8d5fe8e94a7dac as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/66dd2631d6674a9caa8d5fe8e94a7dac 2024-11-16T14:53:36,786 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/66dd2631d6674a9caa8d5fe8e94a7dac, entries=150, sequenceid=307, filesize=12.0 K 2024-11-16T14:53:36,787 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for d96afd6d0e8810c274529387dcd006b9 in 934ms, sequenceid=307, compaction requested=false 2024-11-16T14:53:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-16T14:53:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-16T14:53:36,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-16T14:53:36,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0920 sec 2024-11-16T14:53:36,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.1050 sec 2024-11-16T14:53:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T14:53:36,841 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:36,841 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-16T14:53:36,843 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T14:53:36,844 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:36,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T14:53:36,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T14:53:36,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:36,997 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:53:36,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:36,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:36,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:36,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:36,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:36,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/6dd2b477535e41dfb4fb48b93dbabb85 is 50, key is test_row_0/A:col10/1731768816044/Put/seqid=0 2024-11-16T14:53:37,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742304_1480 (size=12301) 2024-11-16T14:53:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T14:53:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:37,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:37,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768817240, max timestamp=1731768817373Num uniques 859; 2024-11-16T14:53:37,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:37,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768817240, max timestamp=1731768817373Num uniques 859; 2024-11-16T14:53:37,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768817240, max timestamp=1731768817373Num uniques 859; 2024-11-16T14:53:37,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:37,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:37,404 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/6dd2b477535e41dfb4fb48b93dbabb85 2024-11-16T14:53:37,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/fae1f61e002f4e19981d83847978198c is 50, key is test_row_0/B:col10/1731768816044/Put/seqid=0 2024-11-16T14:53:37,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742305_1481 (size=12301) 2024-11-16T14:53:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T14:53:37,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768877543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:37,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768877671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:37,814 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/fae1f61e002f4e19981d83847978198c 2024-11-16T14:53:37,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/8c2ecd32cef7412d82f12c1d0c30405a is 50, key is test_row_0/C:col10/1731768816044/Put/seqid=0 2024-11-16T14:53:37,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742306_1482 (size=12301) 2024-11-16T14:53:37,830 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/8c2ecd32cef7412d82f12c1d0c30405a 2024-11-16T14:53:37,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/6dd2b477535e41dfb4fb48b93dbabb85 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/6dd2b477535e41dfb4fb48b93dbabb85 2024-11-16T14:53:37,836 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/6dd2b477535e41dfb4fb48b93dbabb85, entries=150, sequenceid=321, filesize=12.0 K 2024-11-16T14:53:37,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/fae1f61e002f4e19981d83847978198c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/fae1f61e002f4e19981d83847978198c 2024-11-16T14:53:37,842 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/fae1f61e002f4e19981d83847978198c, entries=150, sequenceid=321, filesize=12.0 K 2024-11-16T14:53:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/8c2ecd32cef7412d82f12c1d0c30405a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/8c2ecd32cef7412d82f12c1d0c30405a 2024-11-16T14:53:37,848 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/8c2ecd32cef7412d82f12c1d0c30405a, entries=150, sequenceid=321, filesize=12.0 K 2024-11-16T14:53:37,849 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d96afd6d0e8810c274529387dcd006b9 in 852ms, sequenceid=321, compaction requested=true 2024-11-16T14:53:37,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:37,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:37,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-16T14:53:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-16T14:53:37,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-16T14:53:37,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0070 sec 2024-11-16T14:53:37,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.0140 sec 2024-11-16T14:53:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:37,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T14:53:37,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:37,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:37,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:37,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:37,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:37,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:37,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/bdfbcb1be0384f2d901f4094a8c0862b is 50, key is test_row_0/A:col10/1731768817539/Put/seqid=0 2024-11-16T14:53:37,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742307_1483 (size=14741) 2024-11-16T14:53:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T14:53:37,981 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:37,981 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:37,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768877971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-16T14:53:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:37,982 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:53:37,983 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:37,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:53:38,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768878090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:38,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T14:53:38,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:38,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:38,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:38,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:38,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:38,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T14:53:38,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:38,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:38,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:38,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:38,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:53:38,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/bdfbcb1be0384f2d901f4094a8c0862b 2024-11-16T14:53:38,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768878310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:38,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/3509870819e44088a25dace986d615be is 50, key is test_row_0/B:col10/1731768817539/Put/seqid=0 2024-11-16T14:53:38,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742308_1484 (size=12301) 2024-11-16T14:53:38,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/3509870819e44088a25dace986d615be 2024-11-16T14:53:38,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/2b3ae3788f8a4694a719bfd5106bf497 is 50, key is test_row_0/C:col10/1731768817539/Put/seqid=0 2024-11-16T14:53:38,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742309_1485 (size=12301) 2024-11-16T14:53:38,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/2b3ae3788f8a4694a719bfd5106bf497 2024-11-16T14:53:38,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/bdfbcb1be0384f2d901f4094a8c0862b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/bdfbcb1be0384f2d901f4094a8c0862b 2024-11-16T14:53:38,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/bdfbcb1be0384f2d901f4094a8c0862b, entries=200, sequenceid=358, filesize=14.4 K 2024-11-16T14:53:38,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/3509870819e44088a25dace986d615be as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3509870819e44088a25dace986d615be 2024-11-16T14:53:38,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3509870819e44088a25dace986d615be, entries=150, sequenceid=358, filesize=12.0 K 2024-11-16T14:53:38,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/2b3ae3788f8a4694a719bfd5106bf497 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2b3ae3788f8a4694a719bfd5106bf497 2024-11-16T14:53:38,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2b3ae3788f8a4694a719bfd5106bf497, entries=150, sequenceid=358, filesize=12.0 K 2024-11-16T14:53:38,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for d96afd6d0e8810c274529387dcd006b9 in 492ms, sequenceid=358, compaction requested=true 2024-11-16T14:53:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:38,389 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:53:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:38,389 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:53:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:38,390 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:53:38,390 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:53:38,390 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/B is initiating minor compaction (all files) 2024-11-16T14:53:38,390 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/A is initiating minor compaction (all files) 2024-11-16T14:53:38,390 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/A in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:38,390 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/B in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:38,390 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ecf5daf3dc51416ea01950827a979a10, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b795ea1979044af5b741cab6410e2210, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/fae1f61e002f4e19981d83847978198c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3509870819e44088a25dace986d615be] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=48.7 K 2024-11-16T14:53:38,390 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3ce8ef20b096437caf0eb6dc4d153aae, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fd965ee308dd4f7dbb0cad22c961dac5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/6dd2b477535e41dfb4fb48b93dbabb85, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/bdfbcb1be0384f2d901f4094a8c0862b] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=51.0 K 2024-11-16T14:53:38,391 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ce8ef20b096437caf0eb6dc4d153aae, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731768813986 2024-11-16T14:53:38,391 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ecf5daf3dc51416ea01950827a979a10, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731768813986 2024-11-16T14:53:38,391 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd965ee308dd4f7dbb0cad22c961dac5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731768815566 2024-11-16T14:53:38,391 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting b795ea1979044af5b741cab6410e2210, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731768815566 2024-11-16T14:53:38,391 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dd2b477535e41dfb4fb48b93dbabb85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731768815987 2024-11-16T14:53:38,391 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting fae1f61e002f4e19981d83847978198c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731768815987 2024-11-16T14:53:38,391 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdfbcb1be0384f2d901f4094a8c0862b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731768817482 2024-11-16T14:53:38,391 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3509870819e44088a25dace986d615be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731768817482 2024-11-16T14:53:38,402 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#B#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:38,402 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/ca302ff7da604feb93f6f93a111c313a is 50, key is test_row_0/B:col10/1731768817539/Put/seqid=0 2024-11-16T14:53:38,415 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:38,416 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/2f0c432d46c74b3ab597277fc2728b40 is 50, key is test_row_0/A:col10/1731768817539/Put/seqid=0 2024-11-16T14:53:38,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742310_1486 (size=13051) 2024-11-16T14:53:38,422 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/ca302ff7da604feb93f6f93a111c313a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ca302ff7da604feb93f6f93a111c313a 2024-11-16T14:53:38,429 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/B of d96afd6d0e8810c274529387dcd006b9 into ca302ff7da604feb93f6f93a111c313a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:38,429 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:38,429 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/B, priority=12, startTime=1731768818389; duration=0sec 2024-11-16T14:53:38,429 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:38,429 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:B 2024-11-16T14:53:38,429 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:53:38,430 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:53:38,430 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/C is initiating minor compaction (all files) 2024-11-16T14:53:38,430 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/C in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:38,430 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/bd72aac844844057bf02a10c27689275, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/66dd2631d6674a9caa8d5fe8e94a7dac, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/8c2ecd32cef7412d82f12c1d0c30405a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2b3ae3788f8a4694a719bfd5106bf497] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=48.7 K 2024-11-16T14:53:38,431 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting bd72aac844844057bf02a10c27689275, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731768813986 2024-11-16T14:53:38,431 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 66dd2631d6674a9caa8d5fe8e94a7dac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731768815566 2024-11-16T14:53:38,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742311_1487 (size=13051) 2024-11-16T14:53:38,432 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c2ecd32cef7412d82f12c1d0c30405a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731768815987 2024-11-16T14:53:38,432 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b3ae3788f8a4694a719bfd5106bf497, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731768817482 2024-11-16T14:53:38,439 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/2f0c432d46c74b3ab597277fc2728b40 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/2f0c432d46c74b3ab597277fc2728b40 2024-11-16T14:53:38,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T14:53:38,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:38,443 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:53:38,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:38,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:38,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:38,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:38,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:38,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:38,444 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/A of d96afd6d0e8810c274529387dcd006b9 into 2f0c432d46c74b3ab597277fc2728b40(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:38,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:38,444 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/A, priority=12, startTime=1731768818389; duration=0sec 2024-11-16T14:53:38,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:38,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:A 2024-11-16T14:53:38,445 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:38,446 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/04c2714c1bca46dc8da1f0be0b5f7fe6 is 50, key is test_row_0/C:col10/1731768817539/Put/seqid=0 2024-11-16T14:53:38,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/d7051fc9016c41858031da4d500b0fc0 is 50, key is test_row_0/A:col10/1731768817951/Put/seqid=0 2024-11-16T14:53:38,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742312_1488 (size=13051) 2024-11-16T14:53:38,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742313_1489 (size=12301) 2024-11-16T14:53:38,486 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/d7051fc9016c41858031da4d500b0fc0 2024-11-16T14:53:38,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/615e66b645814d88aa74d6313ae96ca9 is 50, key is test_row_0/B:col10/1731768817951/Put/seqid=0 2024-11-16T14:53:38,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742314_1490 (size=12301) 2024-11-16T14:53:38,516 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/615e66b645814d88aa74d6313ae96ca9 2024-11-16T14:53:38,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/5cb907d517074b6aae46ea5300d2cf7d is 50, key is test_row_0/C:col10/1731768817951/Put/seqid=0 2024-11-16T14:53:38,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742315_1491 (size=12301) 2024-11-16T14:53:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:53:38,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:38,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768818749, max timestamp=1731768818782Num uniques 859; 2024-11-16T14:53:38,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768818749, max timestamp=1731768818782Num uniques 859; 2024-11-16T14:53:38,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:38,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:38,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768818749, max timestamp=1731768818782Num uniques 859; 2024-11-16T14:53:38,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:38,876 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/04c2714c1bca46dc8da1f0be0b5f7fe6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/04c2714c1bca46dc8da1f0be0b5f7fe6 2024-11-16T14:53:38,880 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/C of d96afd6d0e8810c274529387dcd006b9 into 04c2714c1bca46dc8da1f0be0b5f7fe6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:38,880 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:38,880 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/C, priority=12, startTime=1731768818389; duration=0sec 2024-11-16T14:53:38,880 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:38,880 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:C 2024-11-16T14:53:38,975 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/5cb907d517074b6aae46ea5300d2cf7d 2024-11-16T14:53:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/d7051fc9016c41858031da4d500b0fc0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d7051fc9016c41858031da4d500b0fc0 2024-11-16T14:53:38,986 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d7051fc9016c41858031da4d500b0fc0, entries=150, sequenceid=369, filesize=12.0 K 2024-11-16T14:53:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768878974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:38,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/615e66b645814d88aa74d6313ae96ca9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/615e66b645814d88aa74d6313ae96ca9 2024-11-16T14:53:38,993 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/615e66b645814d88aa74d6313ae96ca9, entries=150, sequenceid=369, filesize=12.0 K 2024-11-16T14:53:38,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/5cb907d517074b6aae46ea5300d2cf7d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/5cb907d517074b6aae46ea5300d2cf7d 2024-11-16T14:53:38,997 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/5cb907d517074b6aae46ea5300d2cf7d, entries=150, sequenceid=369, filesize=12.0 K 2024-11-16T14:53:38,998 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d96afd6d0e8810c274529387dcd006b9 in 555ms, sequenceid=369, compaction requested=false 2024-11-16T14:53:38,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:38,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:38,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-16T14:53:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-16T14:53:39,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-16T14:53:39,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-16T14:53:39,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.0250 sec 2024-11-16T14:53:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:39,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-16T14:53:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T14:53:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T14:53:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T14:53:39,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/5fb56931cb3f4ecbb74caa6b32c7fed9 is 50, key is test_row_0/A:col10/1731768819101/Put/seqid=0 2024-11-16T14:53:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T14:53:39,124 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:39,125 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-16T14:53:39,127 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:39,129 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:39,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T14:53:39,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742316_1492 (size=14741) 2024-11-16T14:53:39,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/5fb56931cb3f4ecbb74caa6b32c7fed9 2024-11-16T14:53:39,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/b90a3c142f9a4ef18bcf4fd6f7c619cd is 50, key is test_row_0/B:col10/1731768819101/Put/seqid=0 2024-11-16T14:53:39,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742317_1493 (size=12301) 2024-11-16T14:53:39,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768879187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:39,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:39,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:39,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T14:53:39,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T14:53:39,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:39,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:39,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:39,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768879290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:39,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T14:53:39,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:39,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:39,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T14:53:39,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768879500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:39,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/b90a3c142f9a4ef18bcf4fd6f7c619cd 2024-11-16T14:53:39,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/412fff528e8d4d8a921464422a6006be is 50, key is test_row_0/C:col10/1731768819101/Put/seqid=0 2024-11-16T14:53:39,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742318_1494 (size=12301) 2024-11-16T14:53:39,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/412fff528e8d4d8a921464422a6006be 2024-11-16T14:53:39,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/5fb56931cb3f4ecbb74caa6b32c7fed9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5fb56931cb3f4ecbb74caa6b32c7fed9 2024-11-16T14:53:39,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5fb56931cb3f4ecbb74caa6b32c7fed9, entries=200, sequenceid=409, filesize=14.4 K 2024-11-16T14:53:39,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/b90a3c142f9a4ef18bcf4fd6f7c619cd as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b90a3c142f9a4ef18bcf4fd6f7c619cd 2024-11-16T14:53:39,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b90a3c142f9a4ef18bcf4fd6f7c619cd, entries=150, sequenceid=409, filesize=12.0 K 2024-11-16T14:53:39,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/412fff528e8d4d8a921464422a6006be as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/412fff528e8d4d8a921464422a6006be 2024-11-16T14:53:39,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/412fff528e8d4d8a921464422a6006be, entries=150, sequenceid=409, filesize=12.0 K 2024-11-16T14:53:39,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for d96afd6d0e8810c274529387dcd006b9 in 481ms, sequenceid=409, compaction requested=true 2024-11-16T14:53:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:39,583 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:39,583 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:39,584 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:39,584 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/B is initiating minor compaction (all files) 2024-11-16T14:53:39,584 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/B in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,584 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ca302ff7da604feb93f6f93a111c313a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/615e66b645814d88aa74d6313ae96ca9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b90a3c142f9a4ef18bcf4fd6f7c619cd] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=36.8 K 2024-11-16T14:53:39,584 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:39,584 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/A is initiating minor compaction (all files) 2024-11-16T14:53:39,584 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/A in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,584 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/2f0c432d46c74b3ab597277fc2728b40, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d7051fc9016c41858031da4d500b0fc0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5fb56931cb3f4ecbb74caa6b32c7fed9] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=39.2 K 2024-11-16T14:53:39,585 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f0c432d46c74b3ab597277fc2728b40, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731768817482 2024-11-16T14:53:39,585 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ca302ff7da604feb93f6f93a111c313a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731768817482 2024-11-16T14:53:39,585 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 615e66b645814d88aa74d6313ae96ca9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731768817951 2024-11-16T14:53:39,585 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7051fc9016c41858031da4d500b0fc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731768817951 2024-11-16T14:53:39,585 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fb56931cb3f4ecbb74caa6b32c7fed9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731768818931 2024-11-16T14:53:39,585 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting b90a3c142f9a4ef18bcf4fd6f7c619cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731768818931 2024-11-16T14:53:39,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T14:53:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,588 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-16T14:53:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:39,593 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#B#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:39,593 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/830ba44028bf4437be253ac1de2787e5 is 50, key is test_row_0/B:col10/1731768819101/Put/seqid=0 2024-11-16T14:53:39,596 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#A#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:39,596 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/40d7b1c56a2c45858b3d55b552a27a68 is 50, key is test_row_0/A:col10/1731768819101/Put/seqid=0 2024-11-16T14:53:39,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/3b8fb655210d4b1ebc9d3fdc60f27208 is 50, key is test_row_0/A:col10/1731768819184/Put/seqid=0 2024-11-16T14:53:39,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742319_1495 (size=13153) 2024-11-16T14:53:39,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742320_1496 (size=13153) 2024-11-16T14:53:39,616 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/40d7b1c56a2c45858b3d55b552a27a68 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/40d7b1c56a2c45858b3d55b552a27a68 2024-11-16T14:53:39,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742321_1497 (size=12301) 2024-11-16T14:53:39,626 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/3b8fb655210d4b1ebc9d3fdc60f27208 2024-11-16T14:53:39,628 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/A of d96afd6d0e8810c274529387dcd006b9 into 40d7b1c56a2c45858b3d55b552a27a68(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:39,628 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:39,628 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/A, priority=13, startTime=1731768819583; duration=0sec 2024-11-16T14:53:39,628 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:39,628 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:A 2024-11-16T14:53:39,628 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:39,631 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:39,632 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/C is initiating minor compaction (all files) 2024-11-16T14:53:39,632 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/C in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,632 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/04c2714c1bca46dc8da1f0be0b5f7fe6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/5cb907d517074b6aae46ea5300d2cf7d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/412fff528e8d4d8a921464422a6006be] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=36.8 K 2024-11-16T14:53:39,632 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04c2714c1bca46dc8da1f0be0b5f7fe6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731768817482 2024-11-16T14:53:39,632 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb907d517074b6aae46ea5300d2cf7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731768817951 2024-11-16T14:53:39,633 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 412fff528e8d4d8a921464422a6006be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731768818931 2024-11-16T14:53:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/35f8ebd3d8e3484781203b6f58fdf6d3 is 50, key is test_row_0/B:col10/1731768819184/Put/seqid=0 2024-11-16T14:53:39,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742322_1498 (size=12301) 2024-11-16T14:53:39,645 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/35f8ebd3d8e3484781203b6f58fdf6d3 2024-11-16T14:53:39,651 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:39,651 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/3e4180ee27d04c5ea55ff4dec52a5850 is 50, key is test_row_0/C:col10/1731768819101/Put/seqid=0 2024-11-16T14:53:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/147f244f1a214dd8a82066ddecc0d287 is 50, key is test_row_0/C:col10/1731768819184/Put/seqid=0 2024-11-16T14:53:39,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742324_1500 (size=12301) 2024-11-16T14:53:39,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742323_1499 (size=13153) 2024-11-16T14:53:39,661 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/147f244f1a214dd8a82066ddecc0d287 2024-11-16T14:53:39,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/3b8fb655210d4b1ebc9d3fdc60f27208 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3b8fb655210d4b1ebc9d3fdc60f27208 2024-11-16T14:53:39,675 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/3e4180ee27d04c5ea55ff4dec52a5850 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e4180ee27d04c5ea55ff4dec52a5850 2024-11-16T14:53:39,678 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3b8fb655210d4b1ebc9d3fdc60f27208, entries=150, sequenceid=418, filesize=12.0 K 2024-11-16T14:53:39,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/35f8ebd3d8e3484781203b6f58fdf6d3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/35f8ebd3d8e3484781203b6f58fdf6d3 2024-11-16T14:53:39,680 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/C of d96afd6d0e8810c274529387dcd006b9 into 3e4180ee27d04c5ea55ff4dec52a5850(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:39,680 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:39,680 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/C, priority=13, startTime=1731768819583; duration=0sec 2024-11-16T14:53:39,680 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:39,680 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:C 2024-11-16T14:53:39,684 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/35f8ebd3d8e3484781203b6f58fdf6d3, entries=150, sequenceid=418, filesize=12.0 K 2024-11-16T14:53:39,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/147f244f1a214dd8a82066ddecc0d287 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/147f244f1a214dd8a82066ddecc0d287 2024-11-16T14:53:39,688 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/147f244f1a214dd8a82066ddecc0d287, entries=150, sequenceid=418, filesize=12.0 K 2024-11-16T14:53:39,689 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for d96afd6d0e8810c274529387dcd006b9 in 101ms, sequenceid=418, compaction requested=false 2024-11-16T14:53:39,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:39,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-16T14:53:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-16T14:53:39,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-16T14:53:39,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 562 msec 2024-11-16T14:53:39,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 570 msec 2024-11-16T14:53:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T14:53:39,761 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:39,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-16T14:53:39,763 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:53:39,764 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:39,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:39,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:53:39,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:39,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:39,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:39,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:39,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:39,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:39,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/377c588b53ff453b8f2f895e89d6ef2b is 50, key is test_row_0/A:col10/1731768819824/Put/seqid=0 2024-11-16T14:53:39,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742325_1501 (size=12301) 2024-11-16T14:53:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:53:39,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T14:53:39,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:39,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:39,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:39,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:39,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768819918, max timestamp=1731768819978Num uniques 859; 2024-11-16T14:53:39,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:39,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768819918, max timestamp=1731768819978Num uniques 859; 2024-11-16T14:53:39,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:39,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768819918, max timestamp=1731768819978Num uniques 859; 2024-11-16T14:53:39,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:40,009 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/830ba44028bf4437be253ac1de2787e5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/830ba44028bf4437be253ac1de2787e5 2024-11-16T14:53:40,014 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/B of d96afd6d0e8810c274529387dcd006b9 into 830ba44028bf4437be253ac1de2787e5(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:40,014 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:40,014 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/B, priority=13, startTime=1731768819583; duration=0sec 2024-11-16T14:53:40,014 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:40,014 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:B 2024-11-16T14:53:40,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T14:53:40,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:40,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 359 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768880055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768880061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:53:40,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768880062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 360 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768880180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768880180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768880191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T14:53:40,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:40,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/377c588b53ff453b8f2f895e89d6ef2b 2024-11-16T14:53:40,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/2a5e81b26ac1455997beda64b0a8ab09 is 50, key is test_row_0/B:col10/1731768819824/Put/seqid=0 2024-11-16T14:53:40,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742326_1502 (size=12301) 2024-11-16T14:53:40,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/2a5e81b26ac1455997beda64b0a8ab09 2024-11-16T14:53:40,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/82fd9b2c878840dbad0dbd933b7f4538 is 50, key is test_row_0/C:col10/1731768819824/Put/seqid=0 2024-11-16T14:53:40,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742327_1503 (size=12301) 2024-11-16T14:53:40,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T14:53:40,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:40,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:53:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 361 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768880401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768880401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768880410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T14:53:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T14:53:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:40,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 362 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768880721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768880731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768880731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:40,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/82fd9b2c878840dbad0dbd933b7f4538 2024-11-16T14:53:40,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/377c588b53ff453b8f2f895e89d6ef2b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/377c588b53ff453b8f2f895e89d6ef2b 2024-11-16T14:53:40,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/377c588b53ff453b8f2f895e89d6ef2b, entries=150, sequenceid=430, filesize=12.0 K 2024-11-16T14:53:40,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/2a5e81b26ac1455997beda64b0a8ab09 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/2a5e81b26ac1455997beda64b0a8ab09 2024-11-16T14:53:40,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/2a5e81b26ac1455997beda64b0a8ab09, entries=150, sequenceid=430, filesize=12.0 K 2024-11-16T14:53:40,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/82fd9b2c878840dbad0dbd933b7f4538 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/82fd9b2c878840dbad0dbd933b7f4538 2024-11-16T14:53:40,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/82fd9b2c878840dbad0dbd933b7f4538, entries=150, sequenceid=430, filesize=12.0 K 2024-11-16T14:53:40,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d96afd6d0e8810c274529387dcd006b9 in 919ms, sequenceid=430, compaction requested=true 2024-11-16T14:53:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:40,759 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:40,759 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:40,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:40,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:40,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:40,760 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:40,760 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:40,760 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/B is initiating minor compaction (all files) 2024-11-16T14:53:40,760 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/A is initiating minor compaction (all files) 2024-11-16T14:53:40,760 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/B in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,760 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/A in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,760 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/830ba44028bf4437be253ac1de2787e5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/35f8ebd3d8e3484781203b6f58fdf6d3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/2a5e81b26ac1455997beda64b0a8ab09] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=36.9 K 2024-11-16T14:53:40,760 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/40d7b1c56a2c45858b3d55b552a27a68, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3b8fb655210d4b1ebc9d3fdc60f27208, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/377c588b53ff453b8f2f895e89d6ef2b] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=36.9 K 2024-11-16T14:53:40,760 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 830ba44028bf4437be253ac1de2787e5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731768818931 2024-11-16T14:53:40,761 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40d7b1c56a2c45858b3d55b552a27a68, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731768818931 2024-11-16T14:53:40,761 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f8ebd3d8e3484781203b6f58fdf6d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731768819164 2024-11-16T14:53:40,761 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b8fb655210d4b1ebc9d3fdc60f27208, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731768819164 2024-11-16T14:53:40,761 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a5e81b26ac1455997beda64b0a8ab09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731768819824 2024-11-16T14:53:40,761 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 377c588b53ff453b8f2f895e89d6ef2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731768819824 2024-11-16T14:53:40,766 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#A#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:40,766 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/b789148db8244754bbdb16850cae3a72 is 50, key is test_row_0/A:col10/1731768819824/Put/seqid=0 2024-11-16T14:53:40,767 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#B#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:40,767 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/43b09e991c1a4b61b89c32f7114f3f04 is 50, key is test_row_0/B:col10/1731768819824/Put/seqid=0 2024-11-16T14:53:40,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742328_1504 (size=13255) 2024-11-16T14:53:40,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742329_1505 (size=13255) 2024-11-16T14:53:40,776 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/43b09e991c1a4b61b89c32f7114f3f04 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/43b09e991c1a4b61b89c32f7114f3f04 2024-11-16T14:53:40,779 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/B of d96afd6d0e8810c274529387dcd006b9 into 43b09e991c1a4b61b89c32f7114f3f04(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:40,779 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:40,779 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/B, priority=13, startTime=1731768820759; duration=0sec 2024-11-16T14:53:40,779 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:40,779 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:B 2024-11-16T14:53:40,779 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:40,780 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:40,780 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/C is initiating minor compaction (all files) 2024-11-16T14:53:40,780 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/C in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,780 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e4180ee27d04c5ea55ff4dec52a5850, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/147f244f1a214dd8a82066ddecc0d287, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/82fd9b2c878840dbad0dbd933b7f4538] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=36.9 K 2024-11-16T14:53:40,781 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e4180ee27d04c5ea55ff4dec52a5850, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731768818931 2024-11-16T14:53:40,781 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 147f244f1a214dd8a82066ddecc0d287, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731768819164 2024-11-16T14:53:40,781 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 82fd9b2c878840dbad0dbd933b7f4538, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731768819824 2024-11-16T14:53:40,787 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:40,787 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/ed1f355cf13f4681b2c0f6cdcf5212dc is 50, key is test_row_0/C:col10/1731768819824/Put/seqid=0 2024-11-16T14:53:40,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742330_1506 (size=13255) 2024-11-16T14:53:40,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T14:53:40,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:40,837 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T14:53:40,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:40,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:40,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:40,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:40,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:40,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/ef3fe2dd6f9b4863aa375c38a75ec919 is 50, key is test_row_0/A:col10/1731768820061/Put/seqid=0 2024-11-16T14:53:40,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742331_1507 (size=12301) 2024-11-16T14:53:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:53:41,173 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/b789148db8244754bbdb16850cae3a72 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/b789148db8244754bbdb16850cae3a72 2024-11-16T14:53:41,177 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/A of d96afd6d0e8810c274529387dcd006b9 into b789148db8244754bbdb16850cae3a72(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:41,177 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:41,178 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/A, priority=13, startTime=1731768820759; duration=0sec 2024-11-16T14:53:41,178 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:41,178 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:A 2024-11-16T14:53:41,194 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/ed1f355cf13f4681b2c0f6cdcf5212dc as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/ed1f355cf13f4681b2c0f6cdcf5212dc 2024-11-16T14:53:41,198 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/C of d96afd6d0e8810c274529387dcd006b9 into ed1f355cf13f4681b2c0f6cdcf5212dc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:41,198 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:41,198 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/C, priority=13, startTime=1731768820760; duration=0sec 2024-11-16T14:53:41,199 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:41,199 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:C 2024-11-16T14:53:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:41,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. as already flushing 2024-11-16T14:53:41,254 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/ef3fe2dd6f9b4863aa375c38a75ec919 2024-11-16T14:53:41,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/b6050e08ed4d42d687f93eda8f75b768 is 50, key is test_row_0/B:col10/1731768820061/Put/seqid=0 2024-11-16T14:53:41,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742332_1508 (size=12301) 2024-11-16T14:53:41,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 366 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768881264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768881278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768881278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 367 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768881380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768881401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768881401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 368 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768881600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768881611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768881620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,661 INFO [Thread-1910 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1910. 2024-11-16T14:53:41,661 DEBUG [Thread-1910 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:41,661 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:41,661 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:41,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:41,665 INFO [Thread-1908 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1908. 2024-11-16T14:53:41,665 DEBUG [Thread-1908 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:41,665 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:41,665 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:41,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:41,667 INFO [Thread-1906 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1906. 2024-11-16T14:53:41,667 DEBUG [Thread-1906 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:41,667 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:41,667 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/b6050e08ed4d42d687f93eda8f75b768 2024-11-16T14:53:41,667 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:41,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:41,676 INFO [Thread-1914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1914. 2024-11-16T14:53:41,676 DEBUG [Thread-1914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:41,676 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:41,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/6f2443e3a9a74c45bf354508ceb8bb5c is 50, key is test_row_0/C:col10/1731768820061/Put/seqid=0 2024-11-16T14:53:41,676 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:41,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:41,679 INFO [Thread-1912 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1912. 2024-11-16T14:53:41,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742333_1509 (size=12301) 2024-11-16T14:53:41,679 DEBUG [Thread-1912 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:41,679 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:41,679 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:41,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:41,711 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:53:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:53:41,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 369 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38442 deadline: 1731768881921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38510 deadline: 1731768881921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38508 deadline: 1731768881941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731768881961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,962 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:41,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38478 deadline: 1731768881970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:41,971 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in d96afd6d0e8810c274529387dcd006b9 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 , the old value is region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., hostname=fc3fda29f660,43955,1731768673386, seqNum=2 because the exception is null or not the one we care about 2024-11-16T14:53:42,080 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/6f2443e3a9a74c45bf354508ceb8bb5c 2024-11-16T14:53:42,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/ef3fe2dd6f9b4863aa375c38a75ec919 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/ef3fe2dd6f9b4863aa375c38a75ec919 2024-11-16T14:53:42,086 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/ef3fe2dd6f9b4863aa375c38a75ec919, entries=150, sequenceid=469, filesize=12.0 K 2024-11-16T14:53:42,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/b6050e08ed4d42d687f93eda8f75b768 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b6050e08ed4d42d687f93eda8f75b768 2024-11-16T14:53:42,089 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b6050e08ed4d42d687f93eda8f75b768, entries=150, sequenceid=469, filesize=12.0 K 2024-11-16T14:53:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/6f2443e3a9a74c45bf354508ceb8bb5c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/6f2443e3a9a74c45bf354508ceb8bb5c 2024-11-16T14:53:42,092 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/6f2443e3a9a74c45bf354508ceb8bb5c, entries=150, sequenceid=469, filesize=12.0 K 2024-11-16T14:53:42,093 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for d96afd6d0e8810c274529387dcd006b9 in 1256ms, sequenceid=469, compaction requested=false 2024-11-16T14:53:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-16T14:53:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-16T14:53:42,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-16T14:53:42,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3300 sec 2024-11-16T14:53:42,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.3350 sec 2024-11-16T14:53:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:42,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:53:42,434 INFO [Thread-1901 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1901. 2024-11-16T14:53:42,434 INFO [Thread-1897 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1897. 2024-11-16T14:53:42,434 DEBUG [Thread-1901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:42,434 DEBUG [Thread-1897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:42,434 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:42,434 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:42,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:42,434 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:42,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:42,434 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:42,434 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:42,434 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:42,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:42,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:42,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:42,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:42,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/0fc3895ca1494c23b8a80d2fed410f08 is 50, key is test_row_0/A:col10/1731768821276/Put/seqid=0 2024-11-16T14:53:42,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742334_1510 (size=9857) 2024-11-16T14:53:42,451 INFO [Thread-1895 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1895. 2024-11-16T14:53:42,451 DEBUG [Thread-1895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:42,451 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:42,451 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:42,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:42,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/0fc3895ca1494c23b8a80d2fed410f08 2024-11-16T14:53:42,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/9d8476a66f5f4547aba73553f2e8553e is 50, key is test_row_0/B:col10/1731768821276/Put/seqid=0 2024-11-16T14:53:42,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742335_1511 (size=9857) 2024-11-16T14:53:43,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/9d8476a66f5f4547aba73553f2e8553e 2024-11-16T14:53:43,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/b2f981d7cea24b869e0f01408ea8b9ee is 50, key is test_row_0/C:col10/1731768821276/Put/seqid=0 2024-11-16T14:53:43,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742336_1512 (size=9857) 2024-11-16T14:53:43,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/b2f981d7cea24b869e0f01408ea8b9ee 2024-11-16T14:53:43,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/0fc3895ca1494c23b8a80d2fed410f08 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0fc3895ca1494c23b8a80d2fed410f08 2024-11-16T14:53:43,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0fc3895ca1494c23b8a80d2fed410f08, entries=100, sequenceid=483, filesize=9.6 K 2024-11-16T14:53:43,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/9d8476a66f5f4547aba73553f2e8553e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/9d8476a66f5f4547aba73553f2e8553e 2024-11-16T14:53:43,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/9d8476a66f5f4547aba73553f2e8553e, entries=100, sequenceid=483, filesize=9.6 K 2024-11-16T14:53:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/b2f981d7cea24b869e0f01408ea8b9ee as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/b2f981d7cea24b869e0f01408ea8b9ee 2024-11-16T14:53:43,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/b2f981d7cea24b869e0f01408ea8b9ee, entries=100, sequenceid=483, filesize=9.6 K 2024-11-16T14:53:43,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=6.71 KB/6870 for d96afd6d0e8810c274529387dcd006b9 in 1254ms, sequenceid=483, compaction requested=true 2024-11-16T14:53:43,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:53:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:43,688 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:53:43,688 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d96afd6d0e8810c274529387dcd006b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:53:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:43,688 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:43,688 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:43,688 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/A is initiating minor compaction (all files) 2024-11-16T14:53:43,688 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/B is initiating minor compaction (all files) 2024-11-16T14:53:43,688 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/A in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:43,688 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/B in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:43,689 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/b789148db8244754bbdb16850cae3a72, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/ef3fe2dd6f9b4863aa375c38a75ec919, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0fc3895ca1494c23b8a80d2fed410f08] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=34.6 K 2024-11-16T14:53:43,689 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/43b09e991c1a4b61b89c32f7114f3f04, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b6050e08ed4d42d687f93eda8f75b768, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/9d8476a66f5f4547aba73553f2e8553e] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=34.6 K 2024-11-16T14:53:43,689 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting b789148db8244754bbdb16850cae3a72, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731768819824 2024-11-16T14:53:43,689 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 43b09e991c1a4b61b89c32f7114f3f04, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731768819824 2024-11-16T14:53:43,689 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef3fe2dd6f9b4863aa375c38a75ec919, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731768820054 2024-11-16T14:53:43,689 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting b6050e08ed4d42d687f93eda8f75b768, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731768820054 2024-11-16T14:53:43,689 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc3895ca1494c23b8a80d2fed410f08, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731768821276 2024-11-16T14:53:43,689 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d8476a66f5f4547aba73553f2e8553e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731768821276 2024-11-16T14:53:43,695 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:43,695 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/676104e5d19a454fb2d3c6a32d2d9a62 is 50, key is test_row_0/B:col10/1731768821276/Put/seqid=0 2024-11-16T14:53:43,697 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#A#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:43,698 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/f20fe9d2e52d4902a13ae59a1af8fc5a is 50, key is test_row_0/A:col10/1731768821276/Put/seqid=0 2024-11-16T14:53:43,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742337_1513 (size=13357) 2024-11-16T14:53:43,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742338_1514 (size=13357) 2024-11-16T14:53:43,702 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/676104e5d19a454fb2d3c6a32d2d9a62 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/676104e5d19a454fb2d3c6a32d2d9a62 2024-11-16T14:53:43,705 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/B of d96afd6d0e8810c274529387dcd006b9 into 676104e5d19a454fb2d3c6a32d2d9a62(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:43,706 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:43,706 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/B, priority=13, startTime=1731768823688; duration=0sec 2024-11-16T14:53:43,706 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:53:43,706 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:B 2024-11-16T14:53:43,706 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:53:43,706 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:53:43,706 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): d96afd6d0e8810c274529387dcd006b9/C is initiating minor compaction (all files) 2024-11-16T14:53:43,706 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d96afd6d0e8810c274529387dcd006b9/C in TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:43,706 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/ed1f355cf13f4681b2c0f6cdcf5212dc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/6f2443e3a9a74c45bf354508ceb8bb5c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/b2f981d7cea24b869e0f01408ea8b9ee] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp, totalSize=34.6 K 2024-11-16T14:53:43,707 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ed1f355cf13f4681b2c0f6cdcf5212dc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731768819824 2024-11-16T14:53:43,707 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f2443e3a9a74c45bf354508ceb8bb5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731768820054 2024-11-16T14:53:43,707 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f981d7cea24b869e0f01408ea8b9ee, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731768821276 2024-11-16T14:53:43,712 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d96afd6d0e8810c274529387dcd006b9#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:53:43,712 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/bbe72571a1324abea98db18e4e49fe6b is 50, key is test_row_0/C:col10/1731768821276/Put/seqid=0 2024-11-16T14:53:43,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742339_1515 (size=13357) 2024-11-16T14:53:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T14:53:43,921 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:44,105 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/f20fe9d2e52d4902a13ae59a1af8fc5a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/f20fe9d2e52d4902a13ae59a1af8fc5a 2024-11-16T14:53:44,109 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/A of d96afd6d0e8810c274529387dcd006b9 into f20fe9d2e52d4902a13ae59a1af8fc5a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:44,109 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:44,109 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/A, priority=13, startTime=1731768823688; duration=0sec 2024-11-16T14:53:44,109 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:44,109 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:A 2024-11-16T14:53:44,120 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/bbe72571a1324abea98db18e4e49fe6b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/bbe72571a1324abea98db18e4e49fe6b 2024-11-16T14:53:44,123 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d96afd6d0e8810c274529387dcd006b9/C of d96afd6d0e8810c274529387dcd006b9 into bbe72571a1324abea98db18e4e49fe6b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:53:44,124 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d96afd6d0e8810c274529387dcd006b9: 2024-11-16T14:53:44,124 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., storeName=d96afd6d0e8810c274529387dcd006b9/C, priority=13, startTime=1731768823688; duration=0sec 2024-11-16T14:53:44,124 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:53:44,124 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d96afd6d0e8810c274529387dcd006b9:C 2024-11-16T14:53:52,013 INFO [Thread-1899 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1899. 2024-11-16T14:53:52,013 DEBUG [Thread-1899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:52,013 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:52,013 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:52,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:52,022 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-11-16T14:53:52,023 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:53:52,023 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:52,023 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:52,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 306 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1014 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3041 rows 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1016 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3047 rows 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1016 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3044 rows 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1024 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3071 rows 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1018 2024-11-16T14:53:52,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3051 rows 2024-11-16T14:53:52,023 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:53:52,023 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:53:52,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:52,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:52,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:53:52,024 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:53:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:53:52,027 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768832027"}]},"ts":"1731768832027"} 2024-11-16T14:53:52,029 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:53:52,029 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:53:52,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:53:52,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d96afd6d0e8810c274529387dcd006b9, UNASSIGN}] 2024-11-16T14:53:52,032 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d96afd6d0e8810c274529387dcd006b9, UNASSIGN 2024-11-16T14:53:52,033 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=d96afd6d0e8810c274529387dcd006b9, regionState=CLOSING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:52,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d96afd6d0e8810c274529387dcd006b9, UNASSIGN because future has completed 2024-11-16T14:53:52,034 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:53:52,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:53:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:53:52,186 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:52,186 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:53:52,187 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing d96afd6d0e8810c274529387dcd006b9, disabling compactions & flushes 2024-11-16T14:53:52,187 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:52,187 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:52,187 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. after waiting 0 ms 2024-11-16T14:53:52,187 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:52,187 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing d96afd6d0e8810c274529387dcd006b9 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-16T14:53:52,187 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=A 2024-11-16T14:53:52,187 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:52,187 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=B 2024-11-16T14:53:52,187 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:52,187 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d96afd6d0e8810c274529387dcd006b9, store=C 2024-11-16T14:53:52,187 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:52,190 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/a4041f65dfca44979533b41ad7055a8b is 50, key is test_row_0/A:col10/1731768832021/Put/seqid=0 2024-11-16T14:53:52,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742340_1516 (size=12301) 2024-11-16T14:53:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:53:52,597 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/a4041f65dfca44979533b41ad7055a8b 2024-11-16T14:53:52,603 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/ce11edf80f84430ca70a0b37ea8d8496 is 50, key is test_row_0/B:col10/1731768832021/Put/seqid=0 2024-11-16T14:53:52,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742341_1517 (size=12301) 2024-11-16T14:53:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:53:53,008 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/ce11edf80f84430ca70a0b37ea8d8496 2024-11-16T14:53:53,018 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/9f4455d4e32f4e4a98c5d8c3d908037d is 50, key is test_row_0/C:col10/1731768832021/Put/seqid=0 2024-11-16T14:53:53,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742342_1518 (size=12301) 2024-11-16T14:53:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:53:53,423 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/9f4455d4e32f4e4a98c5d8c3d908037d 2024-11-16T14:53:53,427 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/A/a4041f65dfca44979533b41ad7055a8b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/a4041f65dfca44979533b41ad7055a8b 2024-11-16T14:53:53,431 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/a4041f65dfca44979533b41ad7055a8b, entries=150, sequenceid=492, filesize=12.0 K 2024-11-16T14:53:53,432 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/B/ce11edf80f84430ca70a0b37ea8d8496 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ce11edf80f84430ca70a0b37ea8d8496 2024-11-16T14:53:53,435 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ce11edf80f84430ca70a0b37ea8d8496, entries=150, sequenceid=492, filesize=12.0 K 2024-11-16T14:53:53,436 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/.tmp/C/9f4455d4e32f4e4a98c5d8c3d908037d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/9f4455d4e32f4e4a98c5d8c3d908037d 2024-11-16T14:53:53,440 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/9f4455d4e32f4e4a98c5d8c3d908037d, entries=150, sequenceid=492, filesize=12.0 K 2024-11-16T14:53:53,441 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d96afd6d0e8810c274529387dcd006b9 in 1254ms, sequenceid=492, compaction requested=false 2024-11-16T14:53:53,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/31aaef3bddc8415a97f6c5aedb58a261, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d05a6681007f4fb28ca6171e22cb5332, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/16b5099496c24c6387bcd3248fddb37b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c0dd7d27558c437d832d246b1b1ccb52, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c744fec316544385b36d9ac1fc9428b3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/4ecfc9a714e34022b5e54c9cff7e1c21, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fe33c4019b1d480c8f54fd1c2215b218, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/9c1ca5b666984cfebb0eefb99898ac0a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/8d159299e8284bb2a1512d553f439e9e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/32bc76295a6849adb00f9c4856ac0dce, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/f54595ccbfc448c2ad8c70eea1065392, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/363db5277f66456c928c2feb8b5d8416, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0a9bdb670bc9465298b5ed50a49582c4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5989294762d64cc18678e59e38621955, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/a2a484f136eb471f944fdae976e82921, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/7556fe34d70f4eef8d8abae24f535cd2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/833ae86b74ad492faaf0aec9ebf059fc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/dd703dc7d8e74ff789e8aaa523d2dabb, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3ce8ef20b096437caf0eb6dc4d153aae, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fd965ee308dd4f7dbb0cad22c961dac5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/6dd2b477535e41dfb4fb48b93dbabb85, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/bdfbcb1be0384f2d901f4094a8c0862b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/2f0c432d46c74b3ab597277fc2728b40, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d7051fc9016c41858031da4d500b0fc0, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5fb56931cb3f4ecbb74caa6b32c7fed9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/40d7b1c56a2c45858b3d55b552a27a68, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3b8fb655210d4b1ebc9d3fdc60f27208, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/b789148db8244754bbdb16850cae3a72, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/377c588b53ff453b8f2f895e89d6ef2b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/ef3fe2dd6f9b4863aa375c38a75ec919, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0fc3895ca1494c23b8a80d2fed410f08] to archive 2024-11-16T14:53:53,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:53:53,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/31aaef3bddc8415a97f6c5aedb58a261 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/31aaef3bddc8415a97f6c5aedb58a261 2024-11-16T14:53:53,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d05a6681007f4fb28ca6171e22cb5332 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d05a6681007f4fb28ca6171e22cb5332 2024-11-16T14:53:53,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/16b5099496c24c6387bcd3248fddb37b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/16b5099496c24c6387bcd3248fddb37b 2024-11-16T14:53:53,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c0dd7d27558c437d832d246b1b1ccb52 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c0dd7d27558c437d832d246b1b1ccb52 2024-11-16T14:53:53,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c744fec316544385b36d9ac1fc9428b3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/c744fec316544385b36d9ac1fc9428b3 2024-11-16T14:53:53,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/4ecfc9a714e34022b5e54c9cff7e1c21 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/4ecfc9a714e34022b5e54c9cff7e1c21 2024-11-16T14:53:53,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fe33c4019b1d480c8f54fd1c2215b218 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fe33c4019b1d480c8f54fd1c2215b218 2024-11-16T14:53:53,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/9c1ca5b666984cfebb0eefb99898ac0a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/9c1ca5b666984cfebb0eefb99898ac0a 2024-11-16T14:53:53,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/8d159299e8284bb2a1512d553f439e9e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/8d159299e8284bb2a1512d553f439e9e 2024-11-16T14:53:53,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/32bc76295a6849adb00f9c4856ac0dce to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/32bc76295a6849adb00f9c4856ac0dce 2024-11-16T14:53:53,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/f54595ccbfc448c2ad8c70eea1065392 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/f54595ccbfc448c2ad8c70eea1065392 2024-11-16T14:53:53,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/363db5277f66456c928c2feb8b5d8416 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/363db5277f66456c928c2feb8b5d8416 2024-11-16T14:53:53,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0a9bdb670bc9465298b5ed50a49582c4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0a9bdb670bc9465298b5ed50a49582c4 2024-11-16T14:53:53,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5989294762d64cc18678e59e38621955 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5989294762d64cc18678e59e38621955 2024-11-16T14:53:53,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/a2a484f136eb471f944fdae976e82921 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/a2a484f136eb471f944fdae976e82921 2024-11-16T14:53:53,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/7556fe34d70f4eef8d8abae24f535cd2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/7556fe34d70f4eef8d8abae24f535cd2 2024-11-16T14:53:53,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/833ae86b74ad492faaf0aec9ebf059fc to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/833ae86b74ad492faaf0aec9ebf059fc 2024-11-16T14:53:53,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/dd703dc7d8e74ff789e8aaa523d2dabb to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/dd703dc7d8e74ff789e8aaa523d2dabb 2024-11-16T14:53:53,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3ce8ef20b096437caf0eb6dc4d153aae to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3ce8ef20b096437caf0eb6dc4d153aae 2024-11-16T14:53:53,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fd965ee308dd4f7dbb0cad22c961dac5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/fd965ee308dd4f7dbb0cad22c961dac5 2024-11-16T14:53:53,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/6dd2b477535e41dfb4fb48b93dbabb85 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/6dd2b477535e41dfb4fb48b93dbabb85 2024-11-16T14:53:53,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/bdfbcb1be0384f2d901f4094a8c0862b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/bdfbcb1be0384f2d901f4094a8c0862b 2024-11-16T14:53:53,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/2f0c432d46c74b3ab597277fc2728b40 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/2f0c432d46c74b3ab597277fc2728b40 2024-11-16T14:53:53,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d7051fc9016c41858031da4d500b0fc0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/d7051fc9016c41858031da4d500b0fc0 2024-11-16T14:53:53,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5fb56931cb3f4ecbb74caa6b32c7fed9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/5fb56931cb3f4ecbb74caa6b32c7fed9 2024-11-16T14:53:53,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/40d7b1c56a2c45858b3d55b552a27a68 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/40d7b1c56a2c45858b3d55b552a27a68 2024-11-16T14:53:53,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3b8fb655210d4b1ebc9d3fdc60f27208 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/3b8fb655210d4b1ebc9d3fdc60f27208 2024-11-16T14:53:53,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/b789148db8244754bbdb16850cae3a72 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/b789148db8244754bbdb16850cae3a72 2024-11-16T14:53:53,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/377c588b53ff453b8f2f895e89d6ef2b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/377c588b53ff453b8f2f895e89d6ef2b 2024-11-16T14:53:53,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/ef3fe2dd6f9b4863aa375c38a75ec919 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/ef3fe2dd6f9b4863aa375c38a75ec919 2024-11-16T14:53:53,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0fc3895ca1494c23b8a80d2fed410f08 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/0fc3895ca1494c23b8a80d2fed410f08 2024-11-16T14:53:53,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7bfbfadc4125499d8926a019ae27157b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/f8ceb3780db6409fbacefa80b0cb25f9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/daa5d1fca8bb4c9399183c7425764f34, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3680cd0506da4e5b92e8191e26439d57, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c7ab0b8539ec454eb312e8908985d2d2, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/273a08b8f5a64e64bfc9f6c574cacf36, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/389e4596c6dc49018efa7aeff1e89286, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4a35f70ec70141e4a6482ee12aa3dda5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/6cd1ee21e0344e84987ee39fd83058e1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ce6cc8c0e5284644a02733cb04acfa33, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/abdc529c71394fc3be42c69e64f4b739, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/a563dbf8154b4b94b6369a683a006e09, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7430ed0687044506a5505fd1d710d3f7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4081d05bd99e48ae87b126b9c5d60494, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/d601a722af684861a9103307537360e3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c07bc75bc0414542b7c9756a8af2a703, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c3d55c5666d34784b8af90f1081c29c1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ecf5daf3dc51416ea01950827a979a10, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/055bf1b27fb044c5a6b846b554b0cb43, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b795ea1979044af5b741cab6410e2210, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/fae1f61e002f4e19981d83847978198c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ca302ff7da604feb93f6f93a111c313a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3509870819e44088a25dace986d615be, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/615e66b645814d88aa74d6313ae96ca9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/830ba44028bf4437be253ac1de2787e5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b90a3c142f9a4ef18bcf4fd6f7c619cd, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/35f8ebd3d8e3484781203b6f58fdf6d3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/43b09e991c1a4b61b89c32f7114f3f04, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/2a5e81b26ac1455997beda64b0a8ab09, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b6050e08ed4d42d687f93eda8f75b768, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/9d8476a66f5f4547aba73553f2e8553e] to archive 2024-11-16T14:53:53,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:53:53,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7bfbfadc4125499d8926a019ae27157b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7bfbfadc4125499d8926a019ae27157b 2024-11-16T14:53:53,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/f8ceb3780db6409fbacefa80b0cb25f9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/f8ceb3780db6409fbacefa80b0cb25f9 2024-11-16T14:53:53,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/daa5d1fca8bb4c9399183c7425764f34 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/daa5d1fca8bb4c9399183c7425764f34 2024-11-16T14:53:53,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3680cd0506da4e5b92e8191e26439d57 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3680cd0506da4e5b92e8191e26439d57 2024-11-16T14:53:53,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c7ab0b8539ec454eb312e8908985d2d2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c7ab0b8539ec454eb312e8908985d2d2 2024-11-16T14:53:53,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/273a08b8f5a64e64bfc9f6c574cacf36 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/273a08b8f5a64e64bfc9f6c574cacf36 2024-11-16T14:53:53,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/389e4596c6dc49018efa7aeff1e89286 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/389e4596c6dc49018efa7aeff1e89286 2024-11-16T14:53:53,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4a35f70ec70141e4a6482ee12aa3dda5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4a35f70ec70141e4a6482ee12aa3dda5 2024-11-16T14:53:53,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/6cd1ee21e0344e84987ee39fd83058e1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/6cd1ee21e0344e84987ee39fd83058e1 2024-11-16T14:53:53,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ce6cc8c0e5284644a02733cb04acfa33 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ce6cc8c0e5284644a02733cb04acfa33 2024-11-16T14:53:53,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/abdc529c71394fc3be42c69e64f4b739 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/abdc529c71394fc3be42c69e64f4b739 2024-11-16T14:53:53,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/a563dbf8154b4b94b6369a683a006e09 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/a563dbf8154b4b94b6369a683a006e09 2024-11-16T14:53:53,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7430ed0687044506a5505fd1d710d3f7 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/7430ed0687044506a5505fd1d710d3f7 2024-11-16T14:53:53,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4081d05bd99e48ae87b126b9c5d60494 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/4081d05bd99e48ae87b126b9c5d60494 2024-11-16T14:53:53,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/d601a722af684861a9103307537360e3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/d601a722af684861a9103307537360e3 2024-11-16T14:53:53,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c07bc75bc0414542b7c9756a8af2a703 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c07bc75bc0414542b7c9756a8af2a703 2024-11-16T14:53:53,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c3d55c5666d34784b8af90f1081c29c1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/c3d55c5666d34784b8af90f1081c29c1 2024-11-16T14:53:53,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ecf5daf3dc51416ea01950827a979a10 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ecf5daf3dc51416ea01950827a979a10 2024-11-16T14:53:53,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/055bf1b27fb044c5a6b846b554b0cb43 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/055bf1b27fb044c5a6b846b554b0cb43 2024-11-16T14:53:53,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b795ea1979044af5b741cab6410e2210 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b795ea1979044af5b741cab6410e2210 2024-11-16T14:53:53,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/fae1f61e002f4e19981d83847978198c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/fae1f61e002f4e19981d83847978198c 2024-11-16T14:53:53,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ca302ff7da604feb93f6f93a111c313a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ca302ff7da604feb93f6f93a111c313a 2024-11-16T14:53:53,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3509870819e44088a25dace986d615be to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/3509870819e44088a25dace986d615be 2024-11-16T14:53:53,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/615e66b645814d88aa74d6313ae96ca9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/615e66b645814d88aa74d6313ae96ca9 2024-11-16T14:53:53,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/830ba44028bf4437be253ac1de2787e5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/830ba44028bf4437be253ac1de2787e5 2024-11-16T14:53:53,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b90a3c142f9a4ef18bcf4fd6f7c619cd to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b90a3c142f9a4ef18bcf4fd6f7c619cd 2024-11-16T14:53:53,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/35f8ebd3d8e3484781203b6f58fdf6d3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/35f8ebd3d8e3484781203b6f58fdf6d3 2024-11-16T14:53:53,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/43b09e991c1a4b61b89c32f7114f3f04 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/43b09e991c1a4b61b89c32f7114f3f04 2024-11-16T14:53:53,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/2a5e81b26ac1455997beda64b0a8ab09 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/2a5e81b26ac1455997beda64b0a8ab09 2024-11-16T14:53:53,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b6050e08ed4d42d687f93eda8f75b768 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/b6050e08ed4d42d687f93eda8f75b768 2024-11-16T14:53:53,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/9d8476a66f5f4547aba73553f2e8553e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/9d8476a66f5f4547aba73553f2e8553e 2024-11-16T14:53:53,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/e403ae56e9f64d1eadd27eeef136d19c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d8c13af278cc4ed4a65c5cea8bd99342, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/cbf87f5f4b444add96e750f5a0213f60, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0a8502dcbb5d4bf4a78a67a7e7cf3eff, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a2db803e965044bc9cfaab65fedb42df, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/aa311ef4532f4942931267f34a27bcf3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/00705d7b2dba4ef2b33ea0b88d83f07c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/9bfb912bc95e4a648e0dcb3bc5b33f7c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f0db9c4e0a834815b3c64bc11598a220, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2d7110cb2c9b4cab9b6507904a7121d4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f29c4e91b59449958c84bb6df752421b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e74b80106e9464486fff06db0b0fd2e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/7aa41ca4339d4a0dadae223202f4a700, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a4a6c3b7c3824b5bb17c6fae6ae7bb82, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/ef7bbd0dd34746758d44a69ee0318156, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0ee260eaa8ef4f61ae6d319effac83c5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/609157fdaea44656971b9c829b85e5da, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/bd72aac844844057bf02a10c27689275, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d623d0bd19614f3783d1809b72766423, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/66dd2631d6674a9caa8d5fe8e94a7dac, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/8c2ecd32cef7412d82f12c1d0c30405a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/04c2714c1bca46dc8da1f0be0b5f7fe6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2b3ae3788f8a4694a719bfd5106bf497, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/5cb907d517074b6aae46ea5300d2cf7d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e4180ee27d04c5ea55ff4dec52a5850, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/412fff528e8d4d8a921464422a6006be, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/147f244f1a214dd8a82066ddecc0d287, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/ed1f355cf13f4681b2c0f6cdcf5212dc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/82fd9b2c878840dbad0dbd933b7f4538, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/6f2443e3a9a74c45bf354508ceb8bb5c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/b2f981d7cea24b869e0f01408ea8b9ee] to archive 2024-11-16T14:53:53,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:53:53,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/e403ae56e9f64d1eadd27eeef136d19c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/e403ae56e9f64d1eadd27eeef136d19c 2024-11-16T14:53:53,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d8c13af278cc4ed4a65c5cea8bd99342 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d8c13af278cc4ed4a65c5cea8bd99342 2024-11-16T14:53:53,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/cbf87f5f4b444add96e750f5a0213f60 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/cbf87f5f4b444add96e750f5a0213f60 2024-11-16T14:53:53,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0a8502dcbb5d4bf4a78a67a7e7cf3eff to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0a8502dcbb5d4bf4a78a67a7e7cf3eff 2024-11-16T14:53:53,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a2db803e965044bc9cfaab65fedb42df to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a2db803e965044bc9cfaab65fedb42df 2024-11-16T14:53:53,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/aa311ef4532f4942931267f34a27bcf3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/aa311ef4532f4942931267f34a27bcf3 2024-11-16T14:53:53,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/00705d7b2dba4ef2b33ea0b88d83f07c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/00705d7b2dba4ef2b33ea0b88d83f07c 2024-11-16T14:53:53,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/9bfb912bc95e4a648e0dcb3bc5b33f7c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/9bfb912bc95e4a648e0dcb3bc5b33f7c 2024-11-16T14:53:53,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f0db9c4e0a834815b3c64bc11598a220 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f0db9c4e0a834815b3c64bc11598a220 2024-11-16T14:53:53,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2d7110cb2c9b4cab9b6507904a7121d4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2d7110cb2c9b4cab9b6507904a7121d4 2024-11-16T14:53:53,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f29c4e91b59449958c84bb6df752421b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/f29c4e91b59449958c84bb6df752421b 2024-11-16T14:53:53,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e74b80106e9464486fff06db0b0fd2e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e74b80106e9464486fff06db0b0fd2e 2024-11-16T14:53:53,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/7aa41ca4339d4a0dadae223202f4a700 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/7aa41ca4339d4a0dadae223202f4a700 2024-11-16T14:53:53,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a4a6c3b7c3824b5bb17c6fae6ae7bb82 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/a4a6c3b7c3824b5bb17c6fae6ae7bb82 2024-11-16T14:53:53,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/ef7bbd0dd34746758d44a69ee0318156 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/ef7bbd0dd34746758d44a69ee0318156 2024-11-16T14:53:53,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0ee260eaa8ef4f61ae6d319effac83c5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/0ee260eaa8ef4f61ae6d319effac83c5 2024-11-16T14:53:53,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/609157fdaea44656971b9c829b85e5da to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/609157fdaea44656971b9c829b85e5da 2024-11-16T14:53:53,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/bd72aac844844057bf02a10c27689275 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/bd72aac844844057bf02a10c27689275 2024-11-16T14:53:53,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d623d0bd19614f3783d1809b72766423 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/d623d0bd19614f3783d1809b72766423 2024-11-16T14:53:53,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/66dd2631d6674a9caa8d5fe8e94a7dac to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/66dd2631d6674a9caa8d5fe8e94a7dac 2024-11-16T14:53:53,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/8c2ecd32cef7412d82f12c1d0c30405a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/8c2ecd32cef7412d82f12c1d0c30405a 2024-11-16T14:53:53,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/04c2714c1bca46dc8da1f0be0b5f7fe6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/04c2714c1bca46dc8da1f0be0b5f7fe6 2024-11-16T14:53:53,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2b3ae3788f8a4694a719bfd5106bf497 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/2b3ae3788f8a4694a719bfd5106bf497 2024-11-16T14:53:53,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/5cb907d517074b6aae46ea5300d2cf7d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/5cb907d517074b6aae46ea5300d2cf7d 2024-11-16T14:53:53,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e4180ee27d04c5ea55ff4dec52a5850 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/3e4180ee27d04c5ea55ff4dec52a5850 2024-11-16T14:53:53,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/412fff528e8d4d8a921464422a6006be to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/412fff528e8d4d8a921464422a6006be 2024-11-16T14:53:53,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/147f244f1a214dd8a82066ddecc0d287 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/147f244f1a214dd8a82066ddecc0d287 2024-11-16T14:53:53,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/ed1f355cf13f4681b2c0f6cdcf5212dc to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/ed1f355cf13f4681b2c0f6cdcf5212dc 2024-11-16T14:53:53,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/82fd9b2c878840dbad0dbd933b7f4538 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/82fd9b2c878840dbad0dbd933b7f4538 2024-11-16T14:53:53,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/6f2443e3a9a74c45bf354508ceb8bb5c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/6f2443e3a9a74c45bf354508ceb8bb5c 2024-11-16T14:53:53,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/b2f981d7cea24b869e0f01408ea8b9ee to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/b2f981d7cea24b869e0f01408ea8b9ee 2024-11-16T14:53:53,529 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/recovered.edits/495.seqid, newMaxSeqId=495, maxSeqId=1 2024-11-16T14:53:53,529 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. 2024-11-16T14:53:53,529 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for d96afd6d0e8810c274529387dcd006b9: Waiting for close lock at 1731768832187Running coprocessor pre-close hooks at 1731768832187Disabling compacts and flushes for region at 1731768832187Disabling writes for close at 1731768832187Obtaining lock to block concurrent updates at 1731768832187Preparing flush snapshotting stores in d96afd6d0e8810c274529387dcd006b9 at 1731768832187Finished memstore snapshotting TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731768832187Flushing stores of TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9. at 1731768832188 (+1 ms)Flushing d96afd6d0e8810c274529387dcd006b9/A: creating writer at 1731768832188Flushing d96afd6d0e8810c274529387dcd006b9/A: appending metadata at 1731768832189 (+1 ms)Flushing d96afd6d0e8810c274529387dcd006b9/A: closing flushed file at 1731768832190 (+1 ms)Flushing d96afd6d0e8810c274529387dcd006b9/B: creating writer at 1731768832601 (+411 ms)Flushing d96afd6d0e8810c274529387dcd006b9/B: appending metadata at 1731768832603 (+2 ms)Flushing d96afd6d0e8810c274529387dcd006b9/B: closing flushed file at 1731768832603Flushing d96afd6d0e8810c274529387dcd006b9/C: creating writer at 1731768833014 (+411 ms)Flushing d96afd6d0e8810c274529387dcd006b9/C: appending metadata at 1731768833017 (+3 ms)Flushing d96afd6d0e8810c274529387dcd006b9/C: closing flushed file at 1731768833017Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10899d77: reopening flushed file at 1731768833427 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a155768: reopening flushed file at 1731768833431 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5765ca31: reopening flushed file at 1731768833435 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d96afd6d0e8810c274529387dcd006b9 in 1254ms, sequenceid=492, compaction requested=false at 1731768833441 (+6 ms)Writing region close event to WAL at 1731768833527 (+86 ms)Running coprocessor post-close hooks at 1731768833529 (+2 ms)Closed at 1731768833529 2024-11-16T14:53:53,530 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:53,531 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=d96afd6d0e8810c274529387dcd006b9, regionState=CLOSED 2024-11-16T14:53:53,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:53:53,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-16T14:53:53,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure d96afd6d0e8810c274529387dcd006b9, server=fc3fda29f660,43955,1731768673386 in 1.4990 sec 2024-11-16T14:53:53,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-16T14:53:53,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d96afd6d0e8810c274529387dcd006b9, UNASSIGN in 1.5030 sec 2024-11-16T14:53:53,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-16T14:53:53,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5060 sec 2024-11-16T14:53:53,538 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768833538"}]},"ts":"1731768833538"} 2024-11-16T14:53:53,539 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:53:53,539 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:53:53,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5160 sec 2024-11-16T14:53:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T14:53:54,172 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:54,174 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:53:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:54,180 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:54,182 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T14:53:54,185 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:54,186 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/recovered.edits] 2024-11-16T14:53:54,188 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/a4041f65dfca44979533b41ad7055a8b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/a4041f65dfca44979533b41ad7055a8b 2024-11-16T14:53:54,189 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/f20fe9d2e52d4902a13ae59a1af8fc5a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/A/f20fe9d2e52d4902a13ae59a1af8fc5a 2024-11-16T14:53:54,191 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/676104e5d19a454fb2d3c6a32d2d9a62 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/676104e5d19a454fb2d3c6a32d2d9a62 2024-11-16T14:53:54,192 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ce11edf80f84430ca70a0b37ea8d8496 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/B/ce11edf80f84430ca70a0b37ea8d8496 2024-11-16T14:53:54,194 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/9f4455d4e32f4e4a98c5d8c3d908037d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/9f4455d4e32f4e4a98c5d8c3d908037d 2024-11-16T14:53:54,195 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/bbe72571a1324abea98db18e4e49fe6b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/C/bbe72571a1324abea98db18e4e49fe6b 2024-11-16T14:53:54,196 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/recovered.edits/495.seqid to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9/recovered.edits/495.seqid 2024-11-16T14:53:54,197 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/d96afd6d0e8810c274529387dcd006b9 2024-11-16T14:53:54,197 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:53:54,198 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:54,200 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:53:54,201 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:53:54,202 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:54,202 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:53:54,202 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731768834202"}]},"ts":"9223372036854775807"} 2024-11-16T14:53:54,204 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:53:54,204 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d96afd6d0e8810c274529387dcd006b9, NAME => 'TestAcidGuarantees,,1731768799440.d96afd6d0e8810c274529387dcd006b9.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:53:54,204 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:53:54,204 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731768834204"}]},"ts":"9223372036854775807"} 2024-11-16T14:53:54,205 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:53:54,206 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:54,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-16T14:53:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T14:53:54,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:53:54,297 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:54,306 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 248), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 503), ProcessCount=11 (was 11), AvailableMemoryMB=5480 (was 5562) 2024-11-16T14:53:54,314 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=5480 2024-11-16T14:53:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:53:54,315 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:53:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:54,317 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:53:54,317 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:54,317 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-16T14:53:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:53:54,317 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:53:54,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742343_1519 (size=1061) 2024-11-16T14:53:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:53:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:53:54,725 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d 2024-11-16T14:53:54,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742344_1520 (size=53) 2024-11-16T14:53:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:53:55,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:53:55,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ca54beef0657cc499b531409f1c5b137, disabling compactions & flushes 2024-11-16T14:53:55,135 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:55,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:55,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. after waiting 0 ms 2024-11-16T14:53:55,135 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:55,135 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:55,136 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ca54beef0657cc499b531409f1c5b137: Waiting for close lock at 1731768835135Disabling compacts and flushes for region at 1731768835135Disabling writes for close at 1731768835135Writing region close event to WAL at 1731768835135Closed at 1731768835135 2024-11-16T14:53:55,138 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:53:55,139 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731768835139"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731768835139"}]},"ts":"1731768835139"} 2024-11-16T14:53:55,143 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T14:53:55,143 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:53:55,144 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768835143"}]},"ts":"1731768835143"} 2024-11-16T14:53:55,145 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T14:53:55,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, ASSIGN}] 2024-11-16T14:53:55,146 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, ASSIGN 2024-11-16T14:53:55,146 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, ASSIGN; state=OFFLINE, location=fc3fda29f660,43955,1731768673386; forceNewPlan=false, retain=false 2024-11-16T14:53:55,297 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=ca54beef0657cc499b531409f1c5b137, regionState=OPENING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:55,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, ASSIGN because future has completed 2024-11-16T14:53:55,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:53:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:53:55,455 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:55,455 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:53:55,455 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,455 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:53:55,455 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,455 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,457 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,459 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:53:55,460 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca54beef0657cc499b531409f1c5b137 columnFamilyName A 2024-11-16T14:53:55,460 DEBUG [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:55,460 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(327): Store=ca54beef0657cc499b531409f1c5b137/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:53:55,461 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,462 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:53:55,462 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca54beef0657cc499b531409f1c5b137 columnFamilyName B 2024-11-16T14:53:55,462 DEBUG [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:55,463 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(327): Store=ca54beef0657cc499b531409f1c5b137/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:53:55,463 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,463 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:53:55,464 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca54beef0657cc499b531409f1c5b137 columnFamilyName C 2024-11-16T14:53:55,464 DEBUG [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:55,464 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(327): Store=ca54beef0657cc499b531409f1c5b137/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:53:55,464 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:55,464 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,465 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,465 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,465 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,466 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,466 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:53:55,467 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,468 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:53:55,468 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened ca54beef0657cc499b531409f1c5b137; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74327749, jitterRate=0.10756976902484894}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:53:55,468 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:55,469 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for ca54beef0657cc499b531409f1c5b137: Running coprocessor pre-open hook at 1731768835456Writing region info on filesystem at 1731768835456Initializing all the Stores at 1731768835457 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768835457Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768835457Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768835457Cleaning up temporary data from old regions at 1731768835466 (+9 ms)Running coprocessor post-open hooks at 1731768835468 (+2 ms)Region opened successfully at 1731768835469 (+1 ms) 2024-11-16T14:53:55,469 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., pid=148, masterSystemTime=1731768835451 2024-11-16T14:53:55,470 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:55,471 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:55,471 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=ca54beef0657cc499b531409f1c5b137, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:55,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:53:55,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-16T14:53:55,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 in 174 msec 2024-11-16T14:53:55,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-16T14:53:55,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, ASSIGN in 329 msec 2024-11-16T14:53:55,476 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:53:55,476 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768835476"}]},"ts":"1731768835476"} 2024-11-16T14:53:55,477 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T14:53:55,478 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:53:55,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-16T14:53:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T14:53:56,471 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:56,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59939b53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:56,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:56,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:56,476 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e918f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:56,478 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f99cdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:56,479 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:56,481 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:56,482 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:53:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is fc3fda29f660,35823,1731768672605 2024-11-16T14:53:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b8007db 2024-11-16T14:53:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:53:56,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:53:56,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T14:53:56,485 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:53:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T14:53:56,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742345_1521 (size=1097) 2024-11-16T14:53:56,899 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T14:53:56,899 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T14:53:56,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:53:56,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, REOPEN/MOVE}] 2024-11-16T14:53:56,906 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, REOPEN/MOVE 2024-11-16T14:53:56,907 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=ca54beef0657cc499b531409f1c5b137, regionState=CLOSING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:56,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, REOPEN/MOVE because future has completed 2024-11-16T14:53:56,908 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:53:56,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:53:57,061 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,061 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:53:57,061 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing ca54beef0657cc499b531409f1c5b137, disabling compactions & flushes 2024-11-16T14:53:57,061 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,061 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,061 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. after waiting 0 ms 2024-11-16T14:53:57,061 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,067 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T14:53:57,068 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,068 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for ca54beef0657cc499b531409f1c5b137: Waiting for close lock at 1731768837061Running coprocessor pre-close hooks at 1731768837061Disabling compacts and flushes for region at 1731768837061Disabling writes for close at 1731768837061Writing region close event to WAL at 1731768837062 (+1 ms)Running coprocessor post-close hooks at 1731768837068 (+6 ms)Closed at 1731768837068 2024-11-16T14:53:57,068 WARN [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: ca54beef0657cc499b531409f1c5b137 to self. 2024-11-16T14:53:57,071 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,071 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=ca54beef0657cc499b531409f1c5b137, regionState=CLOSED 2024-11-16T14:53:57,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:53:57,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-16T14:53:57,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 in 166 msec 2024-11-16T14:53:57,078 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, REOPEN/MOVE; state=CLOSED, location=fc3fda29f660,43955,1731768673386; forceNewPlan=false, retain=true 2024-11-16T14:53:57,229 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=ca54beef0657cc499b531409f1c5b137, regionState=OPENING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, REOPEN/MOVE because future has completed 2024-11-16T14:53:57,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:53:57,389 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,390 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:53:57,391 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,391 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:53:57,391 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,391 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,394 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,395 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:53:57,396 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca54beef0657cc499b531409f1c5b137 columnFamilyName A 2024-11-16T14:53:57,397 DEBUG [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:57,398 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(327): Store=ca54beef0657cc499b531409f1c5b137/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:53:57,399 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,399 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:53:57,399 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca54beef0657cc499b531409f1c5b137 columnFamilyName B 2024-11-16T14:53:57,400 DEBUG [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:57,400 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(327): Store=ca54beef0657cc499b531409f1c5b137/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:53:57,400 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,401 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T14:53:57,401 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca54beef0657cc499b531409f1c5b137 columnFamilyName C 2024-11-16T14:53:57,401 DEBUG [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:57,401 INFO [StoreOpener-ca54beef0657cc499b531409f1c5b137-1 {}] regionserver.HStore(327): Store=ca54beef0657cc499b531409f1c5b137/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:53:57,402 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,402 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,402 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,403 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,404 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,404 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,404 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T14:53:57,405 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,406 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened ca54beef0657cc499b531409f1c5b137; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61917067, jitterRate=-0.07736380398273468}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T14:53:57,406 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,406 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for ca54beef0657cc499b531409f1c5b137: Running coprocessor pre-open hook at 1731768837391Writing region info on filesystem at 1731768837391Initializing all the Stores at 1731768837393 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768837393Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768837394 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731768837394Cleaning up temporary data from old regions at 1731768837404 (+10 ms)Running coprocessor post-open hooks at 1731768837406 (+2 ms)Region opened successfully at 1731768837406 2024-11-16T14:53:57,407 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., pid=153, masterSystemTime=1731768837384 2024-11-16T14:53:57,408 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,408 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,409 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=ca54beef0657cc499b531409f1c5b137, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:53:57,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-16T14:53:57,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 in 179 msec 2024-11-16T14:53:57,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-16T14:53:57,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, REOPEN/MOVE in 506 msec 2024-11-16T14:53:57,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-16T14:53:57,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-11-16T14:53:57,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 931 msec 2024-11-16T14:53:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-16T14:53:57,419 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:53:57,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e225a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:57,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:57,421 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15631bf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:57,422 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bab7190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fea92ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:57,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:57,424 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ae8b9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:57,425 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c0ce18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c187dca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:57,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:57,427 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70920842, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:57,428 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2968bf99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76ecb225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:57,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:57,430 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45171b1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:57,431 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3709139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1407fbe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:57,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:57,434 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2261f754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:57,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aaa567f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d844d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:57,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:57,438 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c3c4f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:57,439 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:57,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d368d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533d9664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:57,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:57,442 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a53e09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:57,444 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@773ee2eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383c5fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:57,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:57,446 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af94ef9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:57,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:57,448 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:57,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dda671, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b279a63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:57,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:57,451 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:57,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:57,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@385fff52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:57,453 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:57,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22df5613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fb506a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,35823,-1 for getting cluster id 2024-11-16T14:53:57,454 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:53:57,454 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddbf7b10-fc3f-45cb-abf3-345a89badfe3' 2024-11-16T14:53:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:53:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddbf7b10-fc3f-45cb-abf3-345a89badfe3" 2024-11-16T14:53:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11984980, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,35823,-1] 2024-11-16T14:53:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:53:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:53:57,455 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:53:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79043342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:53:57,456 DEBUG [Thread-2358 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:57,456 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:57,457 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:57,457 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:57,457 DEBUG [Thread-2347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:57,458 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:57,458 DEBUG [Thread-2345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:57,458 DEBUG [Thread-2343 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:57,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:57,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:57,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:57,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:57,459 DEBUG [Thread-2345 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:57,459 DEBUG [Thread-2345 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:57,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:57,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:57,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:57,460 DEBUG [Thread-2350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:57,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:53:57,461 DEBUG [Thread-2356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:53:57,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:53:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:53:57,461 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:53:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,43955,1731768673386, seqNum=-1] 2024-11-16T14:53:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:53:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:53:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:53:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:53:57,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:53:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-16T14:53:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:53:57,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:53:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:53:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:53:57,465 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:53:57,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5] 2024-11-16T14:53:57,467 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:53:57,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:53:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:53:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:53:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:53:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:53:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:53:57,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162a8976d871c34b039ee189e7da2d4097_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768837464/Put/seqid=0 2024-11-16T14:53:57,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768837489, max timestamp=1731768837496Num uniques 859; 2024-11-16T14:53:57,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768837489, max timestamp=1731768837496Num uniques 859; 2024-11-16T14:53:57,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768837489, max timestamp=1731768837496Num uniques 859; 2024-11-16T14:53:57,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:57,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:57,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:53:57,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742346_1522 (size=12154) 2024-11-16T14:53:57,501 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:57,504 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162a8976d871c34b039ee189e7da2d4097_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162a8976d871c34b039ee189e7da2d4097_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:57,504 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/48be10c9dfca48e1a75773195d5a4ea1, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:53:57,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/48be10c9dfca48e1a75773195d5a4ea1 is 175, key is test_row_0/A:col10/1731768837464/Put/seqid=0 2024-11-16T14:53:57,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742347_1523 (size=30955) 2024-11-16T14:53:57,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768897513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768897513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768897514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768897518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768897519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:53:57,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:53:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:53:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:57,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768897620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768897631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768897631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768897631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768897631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:53:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:53:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:57,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:53:57,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768897830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768897841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768897841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768897841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768897841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:57,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/48be10c9dfca48e1a75773195d5a4ea1 2024-11-16T14:53:57,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:53:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:53:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:57,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:57,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/ff86fcc5e3934b3ea8c8f8c7df77ab07 is 50, key is test_row_0/B:col10/1731768837464/Put/seqid=0 2024-11-16T14:53:57,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742348_1524 (size=12001) 2024-11-16T14:53:58,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:53:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:53:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:53:58,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768898140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:58,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768898150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:58,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768898151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768898151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768898151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:58,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:53:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:53:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/ff86fcc5e3934b3ea8c8f8c7df77ab07 2024-11-16T14:53:58,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/1fbfd2cf6f6b4e469cc36539fc1a1b36 is 50, key is test_row_0/C:col10/1731768837464/Put/seqid=0 2024-11-16T14:53:58,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742349_1525 (size=12001) 2024-11-16T14:53:58,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:53:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:53:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:53:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:53:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:53:58,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768898650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:58,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768898661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:58,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768898661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768898661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768898661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:58,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:58,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:58,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:53:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:53:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:53:58,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/1fbfd2cf6f6b4e469cc36539fc1a1b36 2024-11-16T14:53:58,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/48be10c9dfca48e1a75773195d5a4ea1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/48be10c9dfca48e1a75773195d5a4ea1 2024-11-16T14:53:58,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/48be10c9dfca48e1a75773195d5a4ea1, entries=150, sequenceid=15, filesize=30.2 K 2024-11-16T14:53:58,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/ff86fcc5e3934b3ea8c8f8c7df77ab07 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/ff86fcc5e3934b3ea8c8f8c7df77ab07 2024-11-16T14:53:58,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/ff86fcc5e3934b3ea8c8f8c7df77ab07, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T14:53:58,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/1fbfd2cf6f6b4e469cc36539fc1a1b36 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/1fbfd2cf6f6b4e469cc36539fc1a1b36 2024-11-16T14:53:58,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/1fbfd2cf6f6b4e469cc36539fc1a1b36, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T14:53:58,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ca54beef0657cc499b531409f1c5b137 in 1303ms, sequenceid=15, compaction requested=false 2024-11-16T14:53:58,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:53:58,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T14:53:58,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:53:58,839 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T14:53:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:53:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:53:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:53:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:53:58,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111638bc699a4a2f4d3b8a97cd934ba08294_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768837513/Put/seqid=0 2024-11-16T14:53:58,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742350_1526 (size=12154) 2024-11-16T14:53:59,048 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:53:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:53:59,257 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111638bc699a4a2f4d3b8a97cd934ba08294_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111638bc699a4a2f4d3b8a97cd934ba08294_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:59,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/793cb98dced340a9ae12d83e14d58520, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:53:59,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/793cb98dced340a9ae12d83e14d58520 is 175, key is test_row_0/A:col10/1731768837513/Put/seqid=0 2024-11-16T14:53:59,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742351_1527 (size=30955) 2024-11-16T14:53:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:53:59,661 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/793cb98dced340a9ae12d83e14d58520 2024-11-16T14:53:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:53:59,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:53:59,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/a553eebadb94465f93a934239d1b748c is 50, key is test_row_0/B:col10/1731768837513/Put/seqid=0 2024-11-16T14:53:59,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742352_1528 (size=12001) 2024-11-16T14:53:59,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768899677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:59,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768899680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:59,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768899680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:59,687 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:59,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,688 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:53:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768899680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:59,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:59,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768899682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:59,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768899790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:59,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768899790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:53:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:53:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768899790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:53:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:53:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768900001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768900001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768900001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,073 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/a553eebadb94465f93a934239d1b748c 2024-11-16T14:54:00,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/f2046c1a07404f059ed725f26f41c295 is 50, key is test_row_0/C:col10/1731768837513/Put/seqid=0 2024-11-16T14:54:00,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742353_1529 (size=12001) 2024-11-16T14:54:00,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768900310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768900311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768900311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,485 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/f2046c1a07404f059ed725f26f41c295 2024-11-16T14:54:00,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/793cb98dced340a9ae12d83e14d58520 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/793cb98dced340a9ae12d83e14d58520 2024-11-16T14:54:00,491 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/793cb98dced340a9ae12d83e14d58520, entries=150, sequenceid=52, filesize=30.2 K 2024-11-16T14:54:00,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/a553eebadb94465f93a934239d1b748c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/a553eebadb94465f93a934239d1b748c 2024-11-16T14:54:00,495 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/a553eebadb94465f93a934239d1b748c, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T14:54:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/f2046c1a07404f059ed725f26f41c295 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f2046c1a07404f059ed725f26f41c295 2024-11-16T14:54:00,498 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f2046c1a07404f059ed725f26f41c295, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T14:54:00,499 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ca54beef0657cc499b531409f1c5b137 in 1660ms, sequenceid=52, compaction requested=false 2024-11-16T14:54:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:00,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-16T14:54:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-16T14:54:00,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-16T14:54:00,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0330 sec 2024-11-16T14:54:00,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.0410 sec 2024-11-16T14:54:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:00,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T14:54:00,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:00,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:00,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:00,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:00,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:00,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:00,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116700859ed8fc44578b4f0b49d7cefeefa_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768840821/Put/seqid=0 2024-11-16T14:54:00,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742354_1530 (size=12154) 2024-11-16T14:54:00,839 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:00,843 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116700859ed8fc44578b4f0b49d7cefeefa_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116700859ed8fc44578b4f0b49d7cefeefa_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:00,843 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/e2ce2fe938ec485a9a8cc606d645b65f, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:00,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/e2ce2fe938ec485a9a8cc606d645b65f is 175, key is test_row_0/A:col10/1731768840821/Put/seqid=0 2024-11-16T14:54:00,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742355_1531 (size=30955) 2024-11-16T14:54:00,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768840832, max timestamp=1731768840853Num uniques 859; 2024-11-16T14:54:00,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:54:00,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768840832, max timestamp=1731768840853Num uniques 859; 2024-11-16T14:54:00,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:54:00,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731768840832, max timestamp=1731768840853Num uniques 859; 2024-11-16T14:54:00,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T14:54:00,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768900877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768900879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768900880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768900990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768900990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768900990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:01,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768901201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768901201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768901201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:01,247 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/e2ce2fe938ec485a9a8cc606d645b65f 2024-11-16T14:54:01,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/8d5e9dade95942ce92ecc9e55c419771 is 50, key is test_row_0/B:col10/1731768840821/Put/seqid=0 2024-11-16T14:54:01,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742356_1532 (size=12001) 2024-11-16T14:54:01,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768901510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768901510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:01,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768901510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:01,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T14:54:01,631 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:54:01,631 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:54:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-16T14:54:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:54:01,632 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:54:01,633 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:54:01,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:54:01,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/8d5e9dade95942ce92ecc9e55c419771 2024-11-16T14:54:01,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/a0e6e7a1a73c41db84e74b9433469d3a is 50, key is test_row_0/C:col10/1731768840821/Put/seqid=0 2024-11-16T14:54:01,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742357_1533 (size=12001) 2024-11-16T14:54:01,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768901700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:01,705 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:01,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768901710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:01,715 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:54:01,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T14:54:01,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:01,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:01,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:01,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:01,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:01,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T14:54:01,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:01,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:01,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:54:02,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768902021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:02,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768902021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:02,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768902021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:02,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T14:54:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:02,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:02,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/a0e6e7a1a73c41db84e74b9433469d3a 2024-11-16T14:54:02,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/e2ce2fe938ec485a9a8cc606d645b65f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/e2ce2fe938ec485a9a8cc606d645b65f 2024-11-16T14:54:02,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/e2ce2fe938ec485a9a8cc606d645b65f, entries=150, sequenceid=63, filesize=30.2 K 2024-11-16T14:54:02,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/8d5e9dade95942ce92ecc9e55c419771 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/8d5e9dade95942ce92ecc9e55c419771 2024-11-16T14:54:02,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/8d5e9dade95942ce92ecc9e55c419771, entries=150, sequenceid=63, filesize=11.7 K 2024-11-16T14:54:02,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/a0e6e7a1a73c41db84e74b9433469d3a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/a0e6e7a1a73c41db84e74b9433469d3a 2024-11-16T14:54:02,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/a0e6e7a1a73c41db84e74b9433469d3a, entries=150, sequenceid=63, filesize=11.7 K 2024-11-16T14:54:02,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ca54beef0657cc499b531409f1c5b137 in 1289ms, sequenceid=63, compaction requested=true 2024-11-16T14:54:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:54:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:54:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:02,111 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:54:02,111 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:02,112 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:02,112 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:02,112 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/B is initiating minor compaction (all files) 2024-11-16T14:54:02,112 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/A is initiating minor compaction (all files) 2024-11-16T14:54:02,112 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/A in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:02,112 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/B in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:02,112 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/ff86fcc5e3934b3ea8c8f8c7df77ab07, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/a553eebadb94465f93a934239d1b748c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/8d5e9dade95942ce92ecc9e55c419771] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=35.2 K 2024-11-16T14:54:02,112 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/48be10c9dfca48e1a75773195d5a4ea1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/793cb98dced340a9ae12d83e14d58520, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/e2ce2fe938ec485a9a8cc606d645b65f] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=90.7 K 2024-11-16T14:54:02,112 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:02,112 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/48be10c9dfca48e1a75773195d5a4ea1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/793cb98dced340a9ae12d83e14d58520, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/e2ce2fe938ec485a9a8cc606d645b65f] 2024-11-16T14:54:02,112 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ff86fcc5e3934b3ea8c8f8c7df77ab07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731768837464 2024-11-16T14:54:02,112 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48be10c9dfca48e1a75773195d5a4ea1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731768837464 2024-11-16T14:54:02,113 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting a553eebadb94465f93a934239d1b748c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731768837512 2024-11-16T14:54:02,113 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 793cb98dced340a9ae12d83e14d58520, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731768837512 2024-11-16T14:54:02,113 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d5e9dade95942ce92ecc9e55c419771, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731768839673 2024-11-16T14:54:02,113 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ce2fe938ec485a9a8cc606d645b65f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731768839673 2024-11-16T14:54:02,117 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:02,118 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:02,118 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116d8d42aec697d457e8a84b133a945dcfa_ca54beef0657cc499b531409f1c5b137 store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:02,119 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/fa9f00c65f9a4c97ba50bcfbe080e078 is 50, key is test_row_0/B:col10/1731768840821/Put/seqid=0 2024-11-16T14:54:02,120 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116d8d42aec697d457e8a84b133a945dcfa_ca54beef0657cc499b531409f1c5b137, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:02,120 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d8d42aec697d457e8a84b133a945dcfa_ca54beef0657cc499b531409f1c5b137 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:02,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742358_1534 (size=12104) 2024-11-16T14:54:02,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742359_1535 (size=4469) 2024-11-16T14:54:02,128 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#A#compaction#457 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:02,129 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/1b6d05a7e73347d8b908186ef4d405d3 is 175, key is test_row_0/A:col10/1731768840821/Put/seqid=0 2024-11-16T14:54:02,132 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/fa9f00c65f9a4c97ba50bcfbe080e078 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/fa9f00c65f9a4c97ba50bcfbe080e078 2024-11-16T14:54:02,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742360_1536 (size=31058) 2024-11-16T14:54:02,135 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/B of ca54beef0657cc499b531409f1c5b137 into fa9f00c65f9a4c97ba50bcfbe080e078(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:02,135 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:02,136 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/B, priority=13, startTime=1731768842111; duration=0sec 2024-11-16T14:54:02,136 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:02,136 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:B 2024-11-16T14:54:02,136 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:02,137 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:02,137 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/C is initiating minor compaction (all files) 2024-11-16T14:54:02,137 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/C in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:02,137 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/1fbfd2cf6f6b4e469cc36539fc1a1b36, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f2046c1a07404f059ed725f26f41c295, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/a0e6e7a1a73c41db84e74b9433469d3a] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=35.2 K 2024-11-16T14:54:02,137 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fbfd2cf6f6b4e469cc36539fc1a1b36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731768837464 2024-11-16T14:54:02,138 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f2046c1a07404f059ed725f26f41c295, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731768837512 2024-11-16T14:54:02,138 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting a0e6e7a1a73c41db84e74b9433469d3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731768839673 2024-11-16T14:54:02,143 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:02,144 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/3da308a9f5ed4dceb22c32ccb34fb527 is 50, key is test_row_0/C:col10/1731768840821/Put/seqid=0 2024-11-16T14:54:02,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742361_1537 (size=12104) 2024-11-16T14:54:02,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T14:54:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:02,243 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-16T14:54:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:54:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:54:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:54:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fccd101ee57b41e099516ce62e13b624_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768840873/Put/seqid=0 2024-11-16T14:54:02,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742362_1538 (size=12154) 2024-11-16T14:54:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:54:02,536 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/1b6d05a7e73347d8b908186ef4d405d3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1b6d05a7e73347d8b908186ef4d405d3 2024-11-16T14:54:02,540 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/A of ca54beef0657cc499b531409f1c5b137 into 1b6d05a7e73347d8b908186ef4d405d3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:02,540 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:02,540 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/A, priority=13, startTime=1731768842111; duration=0sec 2024-11-16T14:54:02,540 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:02,540 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:A 2024-11-16T14:54:02,555 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/3da308a9f5ed4dceb22c32ccb34fb527 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/3da308a9f5ed4dceb22c32ccb34fb527 2024-11-16T14:54:02,558 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/C of ca54beef0657cc499b531409f1c5b137 into 3da308a9f5ed4dceb22c32ccb34fb527(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:02,558 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:02,558 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/C, priority=13, startTime=1731768842111; duration=0sec 2024-11-16T14:54:02,558 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:02,558 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:C 2024-11-16T14:54:02,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:02,656 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fccd101ee57b41e099516ce62e13b624_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fccd101ee57b41e099516ce62e13b624_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:02,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/cf3571497f474bf4b67cde8882e23b98, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:02,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/cf3571497f474bf4b67cde8882e23b98 is 175, key is test_row_0/A:col10/1731768840873/Put/seqid=0 2024-11-16T14:54:02,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742363_1539 (size=30955) 2024-11-16T14:54:02,667 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/cf3571497f474bf4b67cde8882e23b98 2024-11-16T14:54:02,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/5fc05db689a44df5bf490e8ed6b7b76b is 50, key is test_row_0/B:col10/1731768840873/Put/seqid=0 2024-11-16T14:54:02,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742364_1540 (size=12001) 2024-11-16T14:54:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:54:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:03,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:03,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768903046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768903050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768903051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,085 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/5fc05db689a44df5bf490e8ed6b7b76b 2024-11-16T14:54:03,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/2b4ec4befc094fe494d3c252b727f199 is 50, key is test_row_0/C:col10/1731768840873/Put/seqid=0 2024-11-16T14:54:03,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742365_1541 (size=12001) 2024-11-16T14:54:03,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768903160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768903160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768903161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768903370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768903370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768903370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,495 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/2b4ec4befc094fe494d3c252b727f199 2024-11-16T14:54:03,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/cf3571497f474bf4b67cde8882e23b98 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cf3571497f474bf4b67cde8882e23b98 2024-11-16T14:54:03,501 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cf3571497f474bf4b67cde8882e23b98, entries=150, sequenceid=98, filesize=30.2 K 2024-11-16T14:54:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/5fc05db689a44df5bf490e8ed6b7b76b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/5fc05db689a44df5bf490e8ed6b7b76b 2024-11-16T14:54:03,504 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/5fc05db689a44df5bf490e8ed6b7b76b, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T14:54:03,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/2b4ec4befc094fe494d3c252b727f199 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/2b4ec4befc094fe494d3c252b727f199 2024-11-16T14:54:03,507 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/2b4ec4befc094fe494d3c252b727f199, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T14:54:03,508 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for ca54beef0657cc499b531409f1c5b137 in 1265ms, sequenceid=98, compaction requested=false 2024-11-16T14:54:03,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:03,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:03,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-16T14:54:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-16T14:54:03,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-16T14:54:03,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8760 sec 2024-11-16T14:54:03,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8800 sec 2024-11-16T14:54:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:03,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:54:03,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:03,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:03,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:03,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:03,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:03,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:03,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a8add5867b274f14bbf1f414213d840c_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768843684/Put/seqid=0 2024-11-16T14:54:03,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742366_1542 (size=12154) 2024-11-16T14:54:03,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768903709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768903710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768903711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T14:54:03,781 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:54:03,781 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:54:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-16T14:54:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:54:03,783 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:54:03,783 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:54:03,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:54:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768903820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768903820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768903820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:03,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:54:03,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:03,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:03,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:03,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:03,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:03,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768904030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:04,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768904030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768904031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:04,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:04,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:04,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,094 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:04,096 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a8add5867b274f14bbf1f414213d840c_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a8add5867b274f14bbf1f414213d840c_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:04,097 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/9903be8b525b4b809d12f0066fec6997, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:04,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/9903be8b525b4b809d12f0066fec6997 is 175, key is test_row_0/A:col10/1731768843684/Put/seqid=0 2024-11-16T14:54:04,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742367_1543 (size=30955) 2024-11-16T14:54:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:54:04,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:04,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:04,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768904340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:04,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768904340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:04,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768904340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:04,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:54:04,500 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/9903be8b525b4b809d12f0066fec6997 2024-11-16T14:54:04,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/893e4192bda54f46bc9ed4f587c2e3ba is 50, key is test_row_0/B:col10/1731768843684/Put/seqid=0 2024-11-16T14:54:04,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742368_1544 (size=12001) 2024-11-16T14:54:04,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:04,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:04,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:04,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:04,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:04,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:04,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768904851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:04,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:04,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:04,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768904851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:04,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768904851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:04,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/893e4192bda54f46bc9ed4f587c2e3ba 2024-11-16T14:54:04,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/66bf75b469f3479b9c75afdb2118a03d is 50, key is test_row_0/C:col10/1731768843684/Put/seqid=0 2024-11-16T14:54:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:54:04,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742369_1545 (size=12001) 2024-11-16T14:54:05,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:05,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:05,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:05,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:05,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:05,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:05,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:05,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:05,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:05,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:05,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:05,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:05,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/66bf75b469f3479b9c75afdb2118a03d 2024-11-16T14:54:05,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/9903be8b525b4b809d12f0066fec6997 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/9903be8b525b4b809d12f0066fec6997 2024-11-16T14:54:05,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/9903be8b525b4b809d12f0066fec6997, entries=150, sequenceid=113, filesize=30.2 K 2024-11-16T14:54:05,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/893e4192bda54f46bc9ed4f587c2e3ba as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/893e4192bda54f46bc9ed4f587c2e3ba 2024-11-16T14:54:05,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/893e4192bda54f46bc9ed4f587c2e3ba, entries=150, sequenceid=113, filesize=11.7 K 2024-11-16T14:54:05,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/66bf75b469f3479b9c75afdb2118a03d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/66bf75b469f3479b9c75afdb2118a03d 2024-11-16T14:54:05,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/66bf75b469f3479b9c75afdb2118a03d, entries=150, sequenceid=113, filesize=11.7 K 2024-11-16T14:54:05,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ca54beef0657cc499b531409f1c5b137 in 1650ms, sequenceid=113, compaction requested=true 2024-11-16T14:54:05,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:05,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:54:05,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:05,335 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:05,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:54:05,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:05,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:54:05,335 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:05,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:05,335 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:05,335 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:05,335 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/B is initiating minor compaction (all files) 2024-11-16T14:54:05,335 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/A is initiating minor compaction (all files) 2024-11-16T14:54:05,335 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/A in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,335 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/B in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,336 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1b6d05a7e73347d8b908186ef4d405d3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cf3571497f474bf4b67cde8882e23b98, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/9903be8b525b4b809d12f0066fec6997] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=90.8 K 2024-11-16T14:54:05,336 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/fa9f00c65f9a4c97ba50bcfbe080e078, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/5fc05db689a44df5bf490e8ed6b7b76b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/893e4192bda54f46bc9ed4f587c2e3ba] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=35.3 K 2024-11-16T14:54:05,336 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,336 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1b6d05a7e73347d8b908186ef4d405d3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cf3571497f474bf4b67cde8882e23b98, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/9903be8b525b4b809d12f0066fec6997] 2024-11-16T14:54:05,336 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9f00c65f9a4c97ba50bcfbe080e078, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731768839673 2024-11-16T14:54:05,336 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b6d05a7e73347d8b908186ef4d405d3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731768839673 2024-11-16T14:54:05,336 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf3571497f474bf4b67cde8882e23b98, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731768840873 2024-11-16T14:54:05,336 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fc05db689a44df5bf490e8ed6b7b76b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731768840873 2024-11-16T14:54:05,336 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9903be8b525b4b809d12f0066fec6997, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731768843047 2024-11-16T14:54:05,336 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 893e4192bda54f46bc9ed4f587c2e3ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731768843047 2024-11-16T14:54:05,340 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:05,341 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:05,341 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/e5d36314103e45d79aaf40e4a8db9b11 is 50, key is test_row_0/B:col10/1731768843684/Put/seqid=0 2024-11-16T14:54:05,343 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411168e4836848c0c45ee91a7481af2f952cb_ca54beef0657cc499b531409f1c5b137 store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:05,345 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411168e4836848c0c45ee91a7481af2f952cb_ca54beef0657cc499b531409f1c5b137, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:05,345 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168e4836848c0c45ee91a7481af2f952cb_ca54beef0657cc499b531409f1c5b137 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:05,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742370_1546 (size=12207) 2024-11-16T14:54:05,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742371_1547 (size=4469) 2024-11-16T14:54:05,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T14:54:05,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,461 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:54:05,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:05,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:05,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:05,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:05,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:05,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:05,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166270198b20a443898038c04ccd573ecf_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768843707/Put/seqid=0 2024-11-16T14:54:05,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742372_1548 (size=12254) 2024-11-16T14:54:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:05,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:05,752 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/e5d36314103e45d79aaf40e4a8db9b11 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/e5d36314103e45d79aaf40e4a8db9b11 2024-11-16T14:54:05,756 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/B of ca54beef0657cc499b531409f1c5b137 into e5d36314103e45d79aaf40e4a8db9b11(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:05,756 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:05,756 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/B, priority=13, startTime=1731768845335; duration=0sec 2024-11-16T14:54:05,756 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:05,756 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:B 2024-11-16T14:54:05,756 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:05,757 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:05,757 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/C is initiating minor compaction (all files) 2024-11-16T14:54:05,757 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/C in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:05,757 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/3da308a9f5ed4dceb22c32ccb34fb527, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/2b4ec4befc094fe494d3c252b727f199, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/66bf75b469f3479b9c75afdb2118a03d] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=35.3 K 2024-11-16T14:54:05,758 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da308a9f5ed4dceb22c32ccb34fb527, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731768839673 2024-11-16T14:54:05,758 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b4ec4befc094fe494d3c252b727f199, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731768840873 2024-11-16T14:54:05,758 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 66bf75b469f3479b9c75afdb2118a03d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731768843047 2024-11-16T14:54:05,766 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#A#compaction#466 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:05,766 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/aad7d1eddedb4b93b04e76f45bb2d417 is 175, key is test_row_0/A:col10/1731768843684/Put/seqid=0 2024-11-16T14:54:05,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:05,776 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#C#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768905771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:05,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742373_1549 (size=31161) 2024-11-16T14:54:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:05,777 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/ebe4bde8199a45ccab04624eb32647f8 is 50, key is test_row_0/C:col10/1731768843684/Put/seqid=0 2024-11-16T14:54:05,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742374_1550 (size=12207) 2024-11-16T14:54:05,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768905775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:05,781 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/aad7d1eddedb4b93b04e76f45bb2d417 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/aad7d1eddedb4b93b04e76f45bb2d417 2024-11-16T14:54:05,785 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/ebe4bde8199a45ccab04624eb32647f8 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/ebe4bde8199a45ccab04624eb32647f8 2024-11-16T14:54:05,786 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/A of ca54beef0657cc499b531409f1c5b137 into aad7d1eddedb4b93b04e76f45bb2d417(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:05,786 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:05,786 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/A, priority=13, startTime=1731768845335; duration=0sec 2024-11-16T14:54:05,786 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:05,786 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:A 2024-11-16T14:54:05,789 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/C of ca54beef0657cc499b531409f1c5b137 into ebe4bde8199a45ccab04624eb32647f8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:05,789 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:05,789 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/C, priority=13, startTime=1731768845335; duration=0sec 2024-11-16T14:54:05,789 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:05,790 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:C 2024-11-16T14:54:05,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768905861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:05,863 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:05,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:05,872 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166270198b20a443898038c04ccd573ecf_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166270198b20a443898038c04ccd573ecf_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768905870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:05,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/70c40893ad20458e919f898c0c88868b, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768905870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:05,873 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:05,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/70c40893ad20458e919f898c0c88868b is 175, key is test_row_0/A:col10/1731768843707/Put/seqid=0 2024-11-16T14:54:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:05,874 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:05,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742375_1551 (size=31055) 2024-11-16T14:54:05,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768905880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:05,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768905890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:54:06,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768906090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:06,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768906100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:06,279 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/70c40893ad20458e919f898c0c88868b 2024-11-16T14:54:06,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/2acaad00e91a452e9d0a643b750a5128 is 50, key is test_row_0/B:col10/1731768843707/Put/seqid=0 2024-11-16T14:54:06,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742376_1552 (size=12101) 2024-11-16T14:54:06,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768906400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:06,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768906411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:06,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:06,708 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/2acaad00e91a452e9d0a643b750a5128 2024-11-16T14:54:06,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/547785effa294d9eb6454b1b8097f98c is 50, key is test_row_0/C:col10/1731768843707/Put/seqid=0 2024-11-16T14:54:06,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742377_1553 (size=12101) 2024-11-16T14:54:06,718 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/547785effa294d9eb6454b1b8097f98c 2024-11-16T14:54:06,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/70c40893ad20458e919f898c0c88868b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/70c40893ad20458e919f898c0c88868b 2024-11-16T14:54:06,724 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/70c40893ad20458e919f898c0c88868b, entries=150, sequenceid=132, filesize=30.3 K 2024-11-16T14:54:06,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/2acaad00e91a452e9d0a643b750a5128 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/2acaad00e91a452e9d0a643b750a5128 2024-11-16T14:54:06,728 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/2acaad00e91a452e9d0a643b750a5128, entries=150, sequenceid=132, filesize=11.8 K 2024-11-16T14:54:06,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/547785effa294d9eb6454b1b8097f98c as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/547785effa294d9eb6454b1b8097f98c 2024-11-16T14:54:06,731 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/547785effa294d9eb6454b1b8097f98c, entries=150, sequenceid=132, filesize=11.8 K 2024-11-16T14:54:06,732 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ca54beef0657cc499b531409f1c5b137 in 1272ms, sequenceid=132, compaction requested=false 2024-11-16T14:54:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-16T14:54:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-16T14:54:06,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-16T14:54:06,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9500 sec 2024-11-16T14:54:06,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.9540 sec 2024-11-16T14:54:06,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:06,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:54:06,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:06,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:06,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:06,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:06,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:06,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:06,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116482de9229c7a4440814dcb2e6377237a_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768846911/Put/seqid=0 2024-11-16T14:54:06,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742378_1554 (size=14794) 2024-11-16T14:54:06,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768906955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:06,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768906958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:07,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768907071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:07,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768907071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:07,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:07,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:07,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768907280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:07,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768907281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:07,339 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:07,342 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116482de9229c7a4440814dcb2e6377237a_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116482de9229c7a4440814dcb2e6377237a_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:07,343 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/1526f7f8e24546a485fe59b69141297d, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:07,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/1526f7f8e24546a485fe59b69141297d is 175, key is test_row_0/A:col10/1731768846911/Put/seqid=0 2024-11-16T14:54:07,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742379_1555 (size=39749) 2024-11-16T14:54:07,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768907590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:07,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768907590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:07,747 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/1526f7f8e24546a485fe59b69141297d 2024-11-16T14:54:07,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/04838ec115c44189848ecdb4c42322f1 is 50, key is test_row_0/B:col10/1731768846911/Put/seqid=0 2024-11-16T14:54:07,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742380_1556 (size=12151) 2024-11-16T14:54:07,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768907880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:07,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768907880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:07,885 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_2' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,885 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:07,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768907890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:07,892 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:07,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:07,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T14:54:07,950 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:54:07,950 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:54:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-16T14:54:07,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:54:07,952 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:54:07,952 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:54:07,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:54:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:54:08,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768908100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:08,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768908100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:08,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T14:54:08,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:08,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:08,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:08,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:08,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:08,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/04838ec115c44189848ecdb4c42322f1 2024-11-16T14:54:08,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/f646b41863054adeb286847dea7fca14 is 50, key is test_row_0/C:col10/1731768846911/Put/seqid=0 2024-11-16T14:54:08,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742381_1557 (size=12151) 2024-11-16T14:54:08,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/f646b41863054adeb286847dea7fca14 2024-11-16T14:54:08,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/1526f7f8e24546a485fe59b69141297d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1526f7f8e24546a485fe59b69141297d 2024-11-16T14:54:08,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1526f7f8e24546a485fe59b69141297d, entries=200, sequenceid=149, filesize=38.8 K 2024-11-16T14:54:08,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/04838ec115c44189848ecdb4c42322f1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/04838ec115c44189848ecdb4c42322f1 2024-11-16T14:54:08,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/04838ec115c44189848ecdb4c42322f1, entries=150, sequenceid=149, filesize=11.9 K 2024-11-16T14:54:08,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/f646b41863054adeb286847dea7fca14 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f646b41863054adeb286847dea7fca14 2024-11-16T14:54:08,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f646b41863054adeb286847dea7fca14, entries=150, sequenceid=149, filesize=11.9 K 2024-11-16T14:54:08,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ca54beef0657cc499b531409f1c5b137 in 1263ms, sequenceid=149, compaction requested=true 2024-11-16T14:54:08,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:08,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:54:08,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:08,177 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:08,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:54:08,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:08,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:54:08,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T14:54:08,177 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:08,177 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:08,177 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/A is initiating minor compaction (all files) 2024-11-16T14:54:08,177 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:08,177 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/A in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:08,177 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/B is initiating minor compaction (all files) 2024-11-16T14:54:08,178 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/B in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:08,178 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/aad7d1eddedb4b93b04e76f45bb2d417, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/70c40893ad20458e919f898c0c88868b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1526f7f8e24546a485fe59b69141297d] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=99.6 K 2024-11-16T14:54:08,178 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/e5d36314103e45d79aaf40e4a8db9b11, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/2acaad00e91a452e9d0a643b750a5128, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/04838ec115c44189848ecdb4c42322f1] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=35.6 K 2024-11-16T14:54:08,178 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:08,178 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/aad7d1eddedb4b93b04e76f45bb2d417, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/70c40893ad20458e919f898c0c88868b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1526f7f8e24546a485fe59b69141297d] 2024-11-16T14:54:08,178 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d36314103e45d79aaf40e4a8db9b11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731768843047 2024-11-16T14:54:08,178 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 2acaad00e91a452e9d0a643b750a5128, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731768843696 2024-11-16T14:54:08,178 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad7d1eddedb4b93b04e76f45bb2d417, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731768843047 2024-11-16T14:54:08,178 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 04838ec115c44189848ecdb4c42322f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731768845725 2024-11-16T14:54:08,178 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c40893ad20458e919f898c0c88868b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731768843696 2024-11-16T14:54:08,178 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1526f7f8e24546a485fe59b69141297d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731768845725 2024-11-16T14:54:08,182 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:08,184 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#B#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:08,184 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/bd024b506d5640a6907890f66e733eb7 is 50, key is test_row_0/B:col10/1731768846911/Put/seqid=0 2024-11-16T14:54:08,186 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411164544939e766e4517bf031dbc389ea4e8_ca54beef0657cc499b531409f1c5b137 store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:08,188 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411164544939e766e4517bf031dbc389ea4e8_ca54beef0657cc499b531409f1c5b137, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:08,188 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164544939e766e4517bf031dbc389ea4e8_ca54beef0657cc499b531409f1c5b137 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:08,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742382_1558 (size=12459) 2024-11-16T14:54:08,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742383_1559 (size=4469) 2024-11-16T14:54:08,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T14:54:08,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:08,257 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T14:54:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:08,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162538c56d14f443d9842528b7aa9411f9_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768846957/Put/seqid=0 2024-11-16T14:54:08,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742384_1560 (size=12304) 2024-11-16T14:54:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:54:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:54:08,594 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/bd024b506d5640a6907890f66e733eb7 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bd024b506d5640a6907890f66e733eb7 2024-11-16T14:54:08,597 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/B of ca54beef0657cc499b531409f1c5b137 into bd024b506d5640a6907890f66e733eb7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:08,597 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:08,597 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/B, priority=13, startTime=1731768848177; duration=0sec 2024-11-16T14:54:08,597 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:08,597 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:B 2024-11-16T14:54:08,597 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:08,598 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:08,598 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/905f6ec1da564cf9a3e44079efe2aea9 is 175, key is test_row_0/A:col10/1731768846911/Put/seqid=0 2024-11-16T14:54:08,599 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:08,599 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/C is initiating minor compaction (all files) 2024-11-16T14:54:08,599 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/C in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:08,599 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/ebe4bde8199a45ccab04624eb32647f8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/547785effa294d9eb6454b1b8097f98c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f646b41863054adeb286847dea7fca14] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=35.6 K 2024-11-16T14:54:08,599 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting ebe4bde8199a45ccab04624eb32647f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731768843047 2024-11-16T14:54:08,600 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 547785effa294d9eb6454b1b8097f98c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731768843696 2024-11-16T14:54:08,600 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f646b41863054adeb286847dea7fca14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731768845725 2024-11-16T14:54:08,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742385_1561 (size=31413) 2024-11-16T14:54:08,606 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/905f6ec1da564cf9a3e44079efe2aea9 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/905f6ec1da564cf9a3e44079efe2aea9 2024-11-16T14:54:08,610 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#C#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:08,610 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/e166488b64184703870105b99ad62e8b is 50, key is test_row_0/C:col10/1731768846911/Put/seqid=0 2024-11-16T14:54:08,612 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/A of ca54beef0657cc499b531409f1c5b137 into 905f6ec1da564cf9a3e44079efe2aea9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:08,613 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:08,613 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/A, priority=13, startTime=1731768848177; duration=0sec 2024-11-16T14:54:08,613 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:08,613 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:A 2024-11-16T14:54:08,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742386_1562 (size=12459) 2024-11-16T14:54:08,619 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/e166488b64184703870105b99ad62e8b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/e166488b64184703870105b99ad62e8b 2024-11-16T14:54:08,622 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/C of ca54beef0657cc499b531409f1c5b137 into e166488b64184703870105b99ad62e8b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:08,622 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:08,622 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/C, priority=13, startTime=1731768848177; duration=0sec 2024-11-16T14:54:08,622 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:08,622 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:C 2024-11-16T14:54:08,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:08,668 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162538c56d14f443d9842528b7aa9411f9_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162538c56d14f443d9842528b7aa9411f9_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/25bd27bfacf045bbae7c882f738c4d8b, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:08,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/25bd27bfacf045bbae7c882f738c4d8b is 175, key is test_row_0/A:col10/1731768846957/Put/seqid=0 2024-11-16T14:54:08,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742387_1563 (size=31105) 2024-11-16T14:54:09,073 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/25bd27bfacf045bbae7c882f738c4d8b 2024-11-16T14:54:09,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/9d0f20cf1e30477db3f74733950204e8 is 50, key is test_row_0/B:col10/1731768846957/Put/seqid=0 2024-11-16T14:54:09,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742388_1564 (size=12151) 2024-11-16T14:54:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:54:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:09,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:09,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768909130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:09,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:09,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768909134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:09,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768909241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:09,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768909241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:09,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768909451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768909451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:09,482 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/9d0f20cf1e30477db3f74733950204e8 2024-11-16T14:54:09,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/6617c769707e44808974ac4b981d04a4 is 50, key is test_row_0/C:col10/1731768846957/Put/seqid=0 2024-11-16T14:54:09,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742389_1565 (size=12151) 2024-11-16T14:54:09,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768909760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:09,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768909760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:09,890 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/6617c769707e44808974ac4b981d04a4 2024-11-16T14:54:09,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/25bd27bfacf045bbae7c882f738c4d8b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/25bd27bfacf045bbae7c882f738c4d8b 2024-11-16T14:54:09,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/25bd27bfacf045bbae7c882f738c4d8b, entries=150, sequenceid=167, filesize=30.4 K 2024-11-16T14:54:09,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/9d0f20cf1e30477db3f74733950204e8 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/9d0f20cf1e30477db3f74733950204e8 2024-11-16T14:54:09,899 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/9d0f20cf1e30477db3f74733950204e8, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T14:54:09,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/6617c769707e44808974ac4b981d04a4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/6617c769707e44808974ac4b981d04a4 2024-11-16T14:54:09,916 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/6617c769707e44808974ac4b981d04a4, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T14:54:09,917 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ca54beef0657cc499b531409f1c5b137 in 1660ms, sequenceid=167, compaction requested=false 2024-11-16T14:54:09,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:09,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:09,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-16T14:54:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-16T14:54:09,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-16T14:54:09,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9660 sec 2024-11-16T14:54:09,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.9690 sec 2024-11-16T14:54:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T14:54:10,101 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:54:10,101 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:54:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-16T14:54:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:54:10,102 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:54:10,103 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:54:10,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:54:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:54:10,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T14:54:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:10,255 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T14:54:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:10,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e7e1fe841e1f4966b0d9ca5eeaca6e13_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768849132/Put/seqid=0 2024-11-16T14:54:10,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742390_1566 (size=12304) 2024-11-16T14:54:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:10,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:10,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768910299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:10,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768910303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:10,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768910410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:10,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768910410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:54:10,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:10,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768910620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768910620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:10,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:10,667 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e7e1fe841e1f4966b0d9ca5eeaca6e13_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e7e1fe841e1f4966b0d9ca5eeaca6e13_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/36452d8720ca4b2e93e2cc4beb0609d6, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:10,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/36452d8720ca4b2e93e2cc4beb0609d6 is 175, key is test_row_0/A:col10/1731768849132/Put/seqid=0 2024-11-16T14:54:10,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742391_1567 (size=31105) 2024-11-16T14:54:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:54:10,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768910931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:10,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768910931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:11,071 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/36452d8720ca4b2e93e2cc4beb0609d6 2024-11-16T14:54:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d1d5bad216cd42d8b509ba61a735a7f6 is 50, key is test_row_0/B:col10/1731768849132/Put/seqid=0 2024-11-16T14:54:11,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742392_1568 (size=12151) 2024-11-16T14:54:11,079 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d1d5bad216cd42d8b509ba61a735a7f6 2024-11-16T14:54:11,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/bf7231bf2dfe4ad7bdf8b6eb9094d189 is 50, key is test_row_0/C:col10/1731768849132/Put/seqid=0 2024-11-16T14:54:11,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742393_1569 (size=12151) 2024-11-16T14:54:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:54:11,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768911440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:11,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768911441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:11,487 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/bf7231bf2dfe4ad7bdf8b6eb9094d189 2024-11-16T14:54:11,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/36452d8720ca4b2e93e2cc4beb0609d6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/36452d8720ca4b2e93e2cc4beb0609d6 2024-11-16T14:54:11,493 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/36452d8720ca4b2e93e2cc4beb0609d6, entries=150, sequenceid=184, filesize=30.4 K 2024-11-16T14:54:11,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d1d5bad216cd42d8b509ba61a735a7f6 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d1d5bad216cd42d8b509ba61a735a7f6 2024-11-16T14:54:11,496 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d1d5bad216cd42d8b509ba61a735a7f6, entries=150, sequenceid=184, filesize=11.9 K 2024-11-16T14:54:11,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/bf7231bf2dfe4ad7bdf8b6eb9094d189 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bf7231bf2dfe4ad7bdf8b6eb9094d189 2024-11-16T14:54:11,499 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bf7231bf2dfe4ad7bdf8b6eb9094d189, entries=150, sequenceid=184, filesize=11.9 K 2024-11-16T14:54:11,500 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ca54beef0657cc499b531409f1c5b137 in 1245ms, sequenceid=184, compaction requested=true 2024-11-16T14:54:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-16T14:54:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-16T14:54:11,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-16T14:54:11,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-16T14:54:11,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4020 sec 2024-11-16T14:54:11,711 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:54:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:11,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:54:11,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:11,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:11,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:11,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:11,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:11,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:11,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166d29f62b214f4ab2a92a99ffa4a023ca_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768850302/Put/seqid=0 2024-11-16T14:54:11,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742394_1570 (size=14794) 2024-11-16T14:54:11,910 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:11,913 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166d29f62b214f4ab2a92a99ffa4a023ca_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166d29f62b214f4ab2a92a99ffa4a023ca_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:11,913 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/ba347b6085cf4a59a20eff09add0355e, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:11,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/ba347b6085cf4a59a20eff09add0355e is 175, key is test_row_0/A:col10/1731768850302/Put/seqid=0 2024-11-16T14:54:11,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742395_1571 (size=39749) 2024-11-16T14:54:11,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768911927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:11,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768911927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:11,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768911931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768912040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768912040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768912040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T14:54:12,251 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:54:12,251 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:54:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-16T14:54:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:54:12,253 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:54:12,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768912250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,254 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:54:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768912250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:54:12,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768912251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,317 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/ba347b6085cf4a59a20eff09add0355e 2024-11-16T14:54:12,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/6b12856e39fc4da18ace378e07ce9e49 is 50, key is test_row_0/B:col10/1731768850302/Put/seqid=0 2024-11-16T14:54:12,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742396_1572 (size=12151) 2024-11-16T14:54:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:54:12,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:54:12,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:12,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768912450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,454 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768912460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,463 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:54:12,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:12,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:12,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:12,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768912560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768912560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768912560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:12,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:54:12,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:54:12,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:12,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:12,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:12,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/6b12856e39fc4da18ace378e07ce9e49 2024-11-16T14:54:12,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/92c953f8f1ca4ae093176bbc7c41e42f is 50, key is test_row_0/C:col10/1731768850302/Put/seqid=0 2024-11-16T14:54:12,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742397_1573 (size=12151) 2024-11-16T14:54:12,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:54:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:12,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:54:13,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:54:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:13,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:13,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768913070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:13,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768913070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768913070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:13,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/92c953f8f1ca4ae093176bbc7c41e42f 2024-11-16T14:54:13,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/ba347b6085cf4a59a20eff09add0355e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/ba347b6085cf4a59a20eff09add0355e 2024-11-16T14:54:13,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/ba347b6085cf4a59a20eff09add0355e, entries=200, sequenceid=203, filesize=38.8 K 2024-11-16T14:54:13,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/6b12856e39fc4da18ace378e07ce9e49 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/6b12856e39fc4da18ace378e07ce9e49 2024-11-16T14:54:13,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/6b12856e39fc4da18ace378e07ce9e49, entries=150, sequenceid=203, filesize=11.9 K 2024-11-16T14:54:13,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/92c953f8f1ca4ae093176bbc7c41e42f as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/92c953f8f1ca4ae093176bbc7c41e42f 2024-11-16T14:54:13,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/92c953f8f1ca4ae093176bbc7c41e42f, entries=150, sequenceid=203, filesize=11.9 K 2024-11-16T14:54:13,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ca54beef0657cc499b531409f1c5b137 in 1249ms, sequenceid=203, compaction requested=true 2024-11-16T14:54:13,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:54:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:13,152 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:54:13,152 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:54:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:54:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:54:13,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:13,153 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133372 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:54:13,153 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:54:13,153 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/B is initiating minor compaction (all files) 2024-11-16T14:54:13,153 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/A is initiating minor compaction (all files) 2024-11-16T14:54:13,153 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/B in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:13,153 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/A in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:13,153 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/905f6ec1da564cf9a3e44079efe2aea9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/25bd27bfacf045bbae7c882f738c4d8b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/36452d8720ca4b2e93e2cc4beb0609d6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/ba347b6085cf4a59a20eff09add0355e] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=130.2 K 2024-11-16T14:54:13,153 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bd024b506d5640a6907890f66e733eb7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/9d0f20cf1e30477db3f74733950204e8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d1d5bad216cd42d8b509ba61a735a7f6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/6b12856e39fc4da18ace378e07ce9e49] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=47.8 K 2024-11-16T14:54:13,153 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:13,153 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/905f6ec1da564cf9a3e44079efe2aea9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/25bd27bfacf045bbae7c882f738c4d8b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/36452d8720ca4b2e93e2cc4beb0609d6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/ba347b6085cf4a59a20eff09add0355e] 2024-11-16T14:54:13,153 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 905f6ec1da564cf9a3e44079efe2aea9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731768845725 2024-11-16T14:54:13,153 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting bd024b506d5640a6907890f66e733eb7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731768845725 2024-11-16T14:54:13,154 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d0f20cf1e30477db3f74733950204e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731768846937 2024-11-16T14:54:13,154 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25bd27bfacf045bbae7c882f738c4d8b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731768846937 2024-11-16T14:54:13,154 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36452d8720ca4b2e93e2cc4beb0609d6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731768849113 2024-11-16T14:54:13,154 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d1d5bad216cd42d8b509ba61a735a7f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731768849113 2024-11-16T14:54:13,154 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b12856e39fc4da18ace378e07ce9e49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731768850298 2024-11-16T14:54:13,154 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba347b6085cf4a59a20eff09add0355e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731768850274 2024-11-16T14:54:13,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T14:54:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:13,169 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T14:54:13,169 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:13,170 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:13,171 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/f3fe7227f16640edb86c3fa0edbe529b is 50, key is test_row_0/B:col10/1731768850302/Put/seqid=0 2024-11-16T14:54:13,172 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411163f45d052a3954ba086911e5c42ff4ea1_ca54beef0657cc499b531409f1c5b137 store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:13,175 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411163f45d052a3954ba086911e5c42ff4ea1_ca54beef0657cc499b531409f1c5b137, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:13,175 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163f45d052a3954ba086911e5c42ff4ea1_ca54beef0657cc499b531409f1c5b137 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:13,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742398_1574 (size=12595) 2024-11-16T14:54:13,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742399_1575 (size=4469) 2024-11-16T14:54:13,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a8aac3337c0048ba95fbe108892d9aae_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768851930/Put/seqid=0 2024-11-16T14:54:13,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742400_1576 (size=12304) 2024-11-16T14:54:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:54:13,579 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/f3fe7227f16640edb86c3fa0edbe529b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/f3fe7227f16640edb86c3fa0edbe529b 2024-11-16T14:54:13,587 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:13,588 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/eb92f47b6f9b43b8b19ede0e90d7123d is 175, key is test_row_0/A:col10/1731768850302/Put/seqid=0 2024-11-16T14:54:13,590 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca54beef0657cc499b531409f1c5b137/B of ca54beef0657cc499b531409f1c5b137 into f3fe7227f16640edb86c3fa0edbe529b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:13,590 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:13,590 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/B, priority=12, startTime=1731768853152; duration=0sec 2024-11-16T14:54:13,590 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:13,590 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:B 2024-11-16T14:54:13,590 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T14:54:13,591 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T14:54:13,591 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/C is initiating minor compaction (all files) 2024-11-16T14:54:13,592 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/C in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:13,592 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/e166488b64184703870105b99ad62e8b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/6617c769707e44808974ac4b981d04a4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bf7231bf2dfe4ad7bdf8b6eb9094d189, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/92c953f8f1ca4ae093176bbc7c41e42f] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=47.8 K 2024-11-16T14:54:13,592 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting e166488b64184703870105b99ad62e8b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731768845725 2024-11-16T14:54:13,592 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 6617c769707e44808974ac4b981d04a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731768846937 2024-11-16T14:54:13,592 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting bf7231bf2dfe4ad7bdf8b6eb9094d189, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731768849113 2024-11-16T14:54:13,592 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c953f8f1ca4ae093176bbc7c41e42f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731768850298 2024-11-16T14:54:13,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:13,598 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a8aac3337c0048ba95fbe108892d9aae_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a8aac3337c0048ba95fbe108892d9aae_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:13,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/c987b9d772da47838caf7cbfc36f76c5, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:13,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/c987b9d772da47838caf7cbfc36f76c5 is 175, key is test_row_0/A:col10/1731768851930/Put/seqid=0 2024-11-16T14:54:13,602 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#C#compaction#489 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:13,603 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/9283178426e34a7e9f0317f30ff6255e is 50, key is test_row_0/C:col10/1731768850302/Put/seqid=0 2024-11-16T14:54:13,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742402_1578 (size=31105) 2024-11-16T14:54:13,606 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/c987b9d772da47838caf7cbfc36f76c5 2024-11-16T14:54:13,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742403_1579 (size=12595) 2024-11-16T14:54:13,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742401_1577 (size=31549) 2024-11-16T14:54:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d7a86a8f0dbf49dd99ed796eb2c6f28a is 50, key is test_row_0/B:col10/1731768851930/Put/seqid=0 2024-11-16T14:54:13,619 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/eb92f47b6f9b43b8b19ede0e90d7123d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/eb92f47b6f9b43b8b19ede0e90d7123d 2024-11-16T14:54:13,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742404_1580 (size=12151) 2024-11-16T14:54:13,620 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d7a86a8f0dbf49dd99ed796eb2c6f28a 2024-11-16T14:54:13,624 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca54beef0657cc499b531409f1c5b137/A of ca54beef0657cc499b531409f1c5b137 into eb92f47b6f9b43b8b19ede0e90d7123d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:13,625 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:13,625 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/A, priority=12, startTime=1731768853151; duration=0sec 2024-11-16T14:54:13,625 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:13,625 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:A 2024-11-16T14:54:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/c5ab065eade94112b46dad0210349616 is 50, key is test_row_0/C:col10/1731768851930/Put/seqid=0 2024-11-16T14:54:13,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742405_1581 (size=12151) 2024-11-16T14:54:14,011 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/9283178426e34a7e9f0317f30ff6255e as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9283178426e34a7e9f0317f30ff6255e 2024-11-16T14:54:14,014 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca54beef0657cc499b531409f1c5b137/C of ca54beef0657cc499b531409f1c5b137 into 9283178426e34a7e9f0317f30ff6255e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:14,014 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:14,014 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/C, priority=12, startTime=1731768853152; duration=0sec 2024-11-16T14:54:14,014 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:14,014 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:C 2024-11-16T14:54:14,038 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/c5ab065eade94112b46dad0210349616 2024-11-16T14:54:14,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/c987b9d772da47838caf7cbfc36f76c5 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/c987b9d772da47838caf7cbfc36f76c5 2024-11-16T14:54:14,044 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/c987b9d772da47838caf7cbfc36f76c5, entries=150, sequenceid=216, filesize=30.4 K 2024-11-16T14:54:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d7a86a8f0dbf49dd99ed796eb2c6f28a as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d7a86a8f0dbf49dd99ed796eb2c6f28a 2024-11-16T14:54:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,047 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d7a86a8f0dbf49dd99ed796eb2c6f28a, entries=150, sequenceid=216, filesize=11.9 K 2024-11-16T14:54:14,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/c5ab065eade94112b46dad0210349616 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/c5ab065eade94112b46dad0210349616 2024-11-16T14:54:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,050 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/c5ab065eade94112b46dad0210349616, entries=150, sequenceid=216, filesize=11.9 K 2024-11-16T14:54:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,051 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for ca54beef0657cc499b531409f1c5b137 in 882ms, sequenceid=216, compaction requested=false 2024-11-16T14:54:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:14,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:14,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-16T14:54:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-16T14:54:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-16T14:54:14,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7980 sec 2024-11-16T14:54:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.8030 sec 2024-11-16T14:54:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T14:54:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:14,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165fefa040b02d4221bfec337af3417473_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768854131/Put/seqid=0 2024-11-16T14:54:14,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742406_1582 (size=12304) 2024-11-16T14:54:14,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768854160, max timestamp=1731768854175Num uniques 859; 2024-11-16T14:54:14,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:54:14,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768854160, max timestamp=1731768854175Num uniques 859; 2024-11-16T14:54:14,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:54:14,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731768854160, max timestamp=1731768854175Num uniques 859; 2024-11-16T14:54:14,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/fc3fda29f660:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T14:54:14,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768914210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768914212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768914213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768914320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768914320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768914330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T14:54:14,400 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:54:14,401 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:54:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-16T14:54:14,402 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:54:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:54:14,403 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:54:14,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:54:14,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44228 deadline: 1731768914471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,475 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_0' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44236 deadline: 1731768914490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,493 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,43955,1731768673386 for 'test_row_1' in ca54beef0657cc499b531409f1c5b137 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-16T14:54:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:54:14,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768914531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768914531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768914540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:14,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T14:54:14,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:14,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:14,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:14,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:14,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:14,560 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165fefa040b02d4221bfec337af3417473_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165fefa040b02d4221bfec337af3417473_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:14,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/4b1e94ff751e4988995ae311f1d7ff09, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:14,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/4b1e94ff751e4988995ae311f1d7ff09 is 175, key is test_row_0/A:col10/1731768854131/Put/seqid=0 2024-11-16T14:54:14,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742407_1583 (size=31105) 2024-11-16T14:54:14,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/4b1e94ff751e4988995ae311f1d7ff09 2024-11-16T14:54:14,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/aa55dd077a354e7389196880e0178988 is 50, key is test_row_0/B:col10/1731768854131/Put/seqid=0 2024-11-16T14:54:14,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742408_1584 (size=12151) 2024-11-16T14:54:14,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/aa55dd077a354e7389196880e0178988 2024-11-16T14:54:14,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/4ac02d5fa3b64242bfd0c97c540f20e4 is 50, key is test_row_0/C:col10/1731768854131/Put/seqid=0 2024-11-16T14:54:14,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742409_1585 (size=12151) 2024-11-16T14:54:14,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T14:54:14,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:14,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:14,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:14,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:14,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:54:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768914840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768914840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768914851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:14,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T14:54:14,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:14,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:14,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:14,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:14,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:14,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/4ac02d5fa3b64242bfd0c97c540f20e4 2024-11-16T14:54:14,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/4b1e94ff751e4988995ae311f1d7ff09 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/4b1e94ff751e4988995ae311f1d7ff09 2024-11-16T14:54:14,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/4b1e94ff751e4988995ae311f1d7ff09, entries=150, sequenceid=230, filesize=30.4 K 2024-11-16T14:54:14,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/aa55dd077a354e7389196880e0178988 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/aa55dd077a354e7389196880e0178988 2024-11-16T14:54:14,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/aa55dd077a354e7389196880e0178988, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T14:54:14,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/4ac02d5fa3b64242bfd0c97c540f20e4 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/4ac02d5fa3b64242bfd0c97c540f20e4 2024-11-16T14:54:14,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/4ac02d5fa3b64242bfd0c97c540f20e4, entries=150, sequenceid=230, filesize=11.9 K 2024-11-16T14:54:14,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ca54beef0657cc499b531409f1c5b137 in 865ms, sequenceid=230, compaction requested=true 2024-11-16T14:54:14,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:14,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:54:14,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:14,996 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:14,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:54:14,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:14,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:54:14,996 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:14,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:14,997 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:14,997 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:14,997 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/B is initiating minor compaction (all files) 2024-11-16T14:54:14,997 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/A is initiating minor compaction (all files) 2024-11-16T14:54:14,997 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/B in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:14,997 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/A in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:14,997 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/f3fe7227f16640edb86c3fa0edbe529b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d7a86a8f0dbf49dd99ed796eb2c6f28a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/aa55dd077a354e7389196880e0178988] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=36.0 K 2024-11-16T14:54:14,997 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/eb92f47b6f9b43b8b19ede0e90d7123d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/c987b9d772da47838caf7cbfc36f76c5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/4b1e94ff751e4988995ae311f1d7ff09] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=91.6 K 2024-11-16T14:54:14,997 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:14,997 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/eb92f47b6f9b43b8b19ede0e90d7123d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/c987b9d772da47838caf7cbfc36f76c5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/4b1e94ff751e4988995ae311f1d7ff09] 2024-11-16T14:54:14,997 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting f3fe7227f16640edb86c3fa0edbe529b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731768850298 2024-11-16T14:54:14,997 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb92f47b6f9b43b8b19ede0e90d7123d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731768850298 2024-11-16T14:54:14,997 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d7a86a8f0dbf49dd99ed796eb2c6f28a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731768851918 2024-11-16T14:54:14,997 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting c987b9d772da47838caf7cbfc36f76c5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731768851918 2024-11-16T14:54:14,998 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting aa55dd077a354e7389196880e0178988, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768854123 2024-11-16T14:54:14,998 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b1e94ff751e4988995ae311f1d7ff09, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768854123 2024-11-16T14:54:15,002 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:15,003 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116eb6750b54efc4703913a1d2c5029f3ac_ca54beef0657cc499b531409f1c5b137 store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:15,003 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#B#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:15,004 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/43ffcdad59c44fc0b2dfb020e2e484cc is 50, key is test_row_0/B:col10/1731768854131/Put/seqid=0 2024-11-16T14:54:15,005 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116eb6750b54efc4703913a1d2c5029f3ac_ca54beef0657cc499b531409f1c5b137, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:15,005 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eb6750b54efc4703913a1d2c5029f3ac_ca54beef0657cc499b531409f1c5b137 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:15,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742410_1586 (size=12697) 2024-11-16T14:54:15,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742411_1587 (size=4469) 2024-11-16T14:54:15,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T14:54:15,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:15,016 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T14:54:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:54:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:54:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T14:54:15,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e5b549da60944cf195b407224b741795_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768854209/Put/seqid=0 2024-11-16T14:54:15,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742412_1588 (size=12454) 2024-11-16T14:54:15,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,027 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e5b549da60944cf195b407224b741795_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e5b549da60944cf195b407224b741795_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:15,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/8a7d566e1fae4492bf7d78648825e7f3, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:15,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/8a7d566e1fae4492bf7d78648825e7f3 is 175, key is test_row_0/A:col10/1731768854209/Put/seqid=0 2024-11-16T14:54:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:54:15,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742413_1589 (size=31255) 2024-11-16T14:54:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:15,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:15,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768915367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768915368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:15,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768915372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:15,411 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/43ffcdad59c44fc0b2dfb020e2e484cc as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/43ffcdad59c44fc0b2dfb020e2e484cc 2024-11-16T14:54:15,413 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:15,413 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/78237100fc5241e29487355de0e95e5b is 175, key is test_row_0/A:col10/1731768854131/Put/seqid=0 2024-11-16T14:54:15,419 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/B of ca54beef0657cc499b531409f1c5b137 into 43ffcdad59c44fc0b2dfb020e2e484cc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:15,419 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:15,419 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/B, priority=13, startTime=1731768854996; duration=0sec 2024-11-16T14:54:15,419 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:15,419 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:B 2024-11-16T14:54:15,419 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:15,421 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:15,421 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/C is initiating minor compaction (all files) 2024-11-16T14:54:15,421 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/C in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:15,421 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9283178426e34a7e9f0317f30ff6255e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/c5ab065eade94112b46dad0210349616, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/4ac02d5fa3b64242bfd0c97c540f20e4] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=36.0 K 2024-11-16T14:54:15,421 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9283178426e34a7e9f0317f30ff6255e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731768850298 2024-11-16T14:54:15,421 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ab065eade94112b46dad0210349616, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731768851918 2024-11-16T14:54:15,422 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac02d5fa3b64242bfd0c97c540f20e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768854123 2024-11-16T14:54:15,429 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#C#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:15,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742414_1590 (size=31651) 2024-11-16T14:54:15,429 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/bbb970fcd6cd4158931ab6cef60f6ace is 50, key is test_row_0/C:col10/1731768854131/Put/seqid=0 2024-11-16T14:54:15,433 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/8a7d566e1fae4492bf7d78648825e7f3 2024-11-16T14:54:15,436 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/78237100fc5241e29487355de0e95e5b as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/78237100fc5241e29487355de0e95e5b 2024-11-16T14:54:15,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742415_1591 (size=12697) 2024-11-16T14:54:15,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d47a6f98b01b4ebfab82d2a254825105 is 50, key is test_row_0/B:col10/1731768854209/Put/seqid=0 2024-11-16T14:54:15,444 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/A of ca54beef0657cc499b531409f1c5b137 into 78237100fc5241e29487355de0e95e5b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:15,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:15,444 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/A, priority=13, startTime=1731768854996; duration=0sec 2024-11-16T14:54:15,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:15,444 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:A 2024-11-16T14:54:15,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742416_1592 (size=12301) 2024-11-16T14:54:15,448 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d47a6f98b01b4ebfab82d2a254825105 2024-11-16T14:54:15,449 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/bbb970fcd6cd4158931ab6cef60f6ace as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bbb970fcd6cd4158931ab6cef60f6ace 2024-11-16T14:54:15,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/aaa7b0a9fd68498488342bad1f248551 is 50, key is test_row_0/C:col10/1731768854209/Put/seqid=0 2024-11-16T14:54:15,455 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/C of ca54beef0657cc499b531409f1c5b137 into bbb970fcd6cd4158931ab6cef60f6ace(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:15,455 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:15,455 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/C, priority=13, startTime=1731768854996; duration=0sec 2024-11-16T14:54:15,455 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:15,455 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:C 2024-11-16T14:54:15,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742417_1593 (size=12301) 2024-11-16T14:54:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768915480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:15,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768915480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:15,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768915480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:54:15,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768915690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768915690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:15,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768915690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:15,857 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/aaa7b0a9fd68498488342bad1f248551 2024-11-16T14:54:15,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/8a7d566e1fae4492bf7d78648825e7f3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/8a7d566e1fae4492bf7d78648825e7f3 2024-11-16T14:54:15,862 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/8a7d566e1fae4492bf7d78648825e7f3, entries=150, sequenceid=266, filesize=30.5 K 2024-11-16T14:54:15,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d47a6f98b01b4ebfab82d2a254825105 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d47a6f98b01b4ebfab82d2a254825105 2024-11-16T14:54:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,866 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d47a6f98b01b4ebfab82d2a254825105, entries=150, sequenceid=266, filesize=12.0 K 2024-11-16T14:54:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/aaa7b0a9fd68498488342bad1f248551 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/aaa7b0a9fd68498488342bad1f248551 2024-11-16T14:54:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,872 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/aaa7b0a9fd68498488342bad1f248551, entries=150, sequenceid=266, filesize=12.0 K 2024-11-16T14:54:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,876 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for ca54beef0657cc499b531409f1c5b137 in 861ms, sequenceid=266, compaction requested=false 2024-11-16T14:54:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-16T14:54:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-16T14:54:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-16T14:54:15,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4750 sec 2024-11-16T14:54:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.4800 sec 2024-11-16T14:54:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T14:54:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:16,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e5744c535bb14f769fb821323719c875_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768856003/Put/seqid=0 2024-11-16T14:54:16,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742418_1594 (size=14994) 2024-11-16T14:54:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768916031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768916032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768916032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768916140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768916140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768916140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768916351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768916351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768916351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,421 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:16,424 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e5744c535bb14f769fb821323719c875_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e5744c535bb14f769fb821323719c875_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:16,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/d063e414852145249e2c4353d8488b34, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:16,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/d063e414852145249e2c4353d8488b34 is 175, key is test_row_0/A:col10/1731768856003/Put/seqid=0 2024-11-16T14:54:16,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742419_1595 (size=39949) 2024-11-16T14:54:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T14:54:16,551 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:54:16,551 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T14:54:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-16T14:54:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T14:54:16,552 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T14:54:16,553 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T14:54:16,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T14:54:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T14:54:16,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768916660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768916660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768916660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:16,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T14:54:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:16,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:16,828 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/d063e414852145249e2c4353d8488b34 2024-11-16T14:54:16,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d24b01036ee84f0ab0c275254256da20 is 50, key is test_row_0/B:col10/1731768856003/Put/seqid=0 2024-11-16T14:54:16,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742420_1596 (size=12301) 2024-11-16T14:54:16,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T14:54:16,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:16,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:16,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:16,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:16,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T14:54:17,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T14:54:17,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:17,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T14:54:17,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:17,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44254 deadline: 1731768917170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:17,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44240 deadline: 1731768917170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:17,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:54:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43955 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44216 deadline: 1731768917170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 , the old value is region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T14:54:17,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., hostname=fc3fda29f660,43955,1731768673386, seqNum=5 because the exception is null or not the one we care about 2024-11-16T14:54:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T14:54:17,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d24b01036ee84f0ab0c275254256da20 2024-11-16T14:54:17,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/9f1b035b39e54220a53aee129d21830d is 50, key is test_row_0/C:col10/1731768856003/Put/seqid=0 2024-11-16T14:54:17,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742421_1597 (size=12301) 2024-11-16T14:54:17,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T14:54:17,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:17,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,461 INFO [Thread-2358 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2358. 2024-11-16T14:54:17,461 DEBUG [Thread-2358 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:54:17,461 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:17,461 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:17,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:17,463 INFO [Thread-2356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2356. 2024-11-16T14:54:17,463 DEBUG [Thread-2356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:54:17,463 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:17,463 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:17,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:17,464 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-11-16T14:54:17,464 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:54:17,464 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:17,464 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:17,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:17,465 INFO [Thread-2350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2350. 2024-11-16T14:54:17,465 DEBUG [Thread-2350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:54:17,465 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:17,465 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:17,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:17,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T14:54:17,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:17,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,468 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-11-16T14:54:17,468 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:54:17,468 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:17,468 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:17,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T14:54:17,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:17,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:54:17,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/9f1b035b39e54220a53aee129d21830d 2024-11-16T14:54:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/d063e414852145249e2c4353d8488b34 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/d063e414852145249e2c4353d8488b34 2024-11-16T14:54:17,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/d063e414852145249e2c4353d8488b34, entries=200, sequenceid=282, filesize=39.0 K 2024-11-16T14:54:17,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/d24b01036ee84f0ab0c275254256da20 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d24b01036ee84f0ab0c275254256da20 2024-11-16T14:54:17,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d24b01036ee84f0ab0c275254256da20, entries=150, sequenceid=282, filesize=12.0 K 2024-11-16T14:54:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/9f1b035b39e54220a53aee129d21830d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9f1b035b39e54220a53aee129d21830d 2024-11-16T14:54:17,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9f1b035b39e54220a53aee129d21830d, entries=150, sequenceid=282, filesize=12.0 K 2024-11-16T14:54:17,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ca54beef0657cc499b531409f1c5b137 in 1666ms, sequenceid=282, compaction requested=true 2024-11-16T14:54:17,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:17,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T14:54:17,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:17,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T14:54:17,669 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:17,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:17,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca54beef0657cc499b531409f1c5b137:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T14:54:17,669 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:17,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:17,670 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:17,670 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:17,670 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/B is initiating minor compaction (all files) 2024-11-16T14:54:17,670 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/A is initiating minor compaction (all files) 2024-11-16T14:54:17,670 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/B in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,670 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/A in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,670 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/43ffcdad59c44fc0b2dfb020e2e484cc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d47a6f98b01b4ebfab82d2a254825105, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d24b01036ee84f0ab0c275254256da20] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=36.4 K 2024-11-16T14:54:17,670 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/78237100fc5241e29487355de0e95e5b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/8a7d566e1fae4492bf7d78648825e7f3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/d063e414852145249e2c4353d8488b34] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=100.4 K 2024-11-16T14:54:17,670 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,670 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. files: [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/78237100fc5241e29487355de0e95e5b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/8a7d566e1fae4492bf7d78648825e7f3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/d063e414852145249e2c4353d8488b34] 2024-11-16T14:54:17,670 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ffcdad59c44fc0b2dfb020e2e484cc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768854123 2024-11-16T14:54:17,670 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78237100fc5241e29487355de0e95e5b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768854123 2024-11-16T14:54:17,671 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d47a6f98b01b4ebfab82d2a254825105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731768854200 2024-11-16T14:54:17,671 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7d566e1fae4492bf7d78648825e7f3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731768854200 2024-11-16T14:54:17,671 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting d24b01036ee84f0ab0c275254256da20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731768855365 2024-11-16T14:54:17,671 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] compactions.Compactor(225): Compacting d063e414852145249e2c4353d8488b34, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731768855365 2024-11-16T14:54:17,676 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:17,676 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/bcf5247e1b194167a14f90f85a835d46 is 50, key is test_row_0/B:col10/1731768856003/Put/seqid=0 2024-11-16T14:54:17,679 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:17,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742422_1598 (size=12949) 2024-11-16T14:54:17,681 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116ac9f7d2fdf6b4e9a966ff5c3ea04a2b1_ca54beef0657cc499b531409f1c5b137 store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T14:54:17,706 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116ac9f7d2fdf6b4e9a966ff5c3ea04a2b1_ca54beef0657cc499b531409f1c5b137, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:17,706 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ac9f7d2fdf6b4e9a966ff5c3ea04a2b1_ca54beef0657cc499b531409f1c5b137 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:17,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742423_1599 (size=4469) 2024-11-16T14:54:17,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43955 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T14:54:17,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:17,776 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T14:54:17,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:17,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:17,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:17,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:17,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:17,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ec484a1abeab4d85b9eaef872d080faf_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768856030/Put/seqid=0 2024-11-16T14:54:17,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742424_1600 (size=12454) 2024-11-16T14:54:17,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:17,790 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ec484a1abeab4d85b9eaef872d080faf_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ec484a1abeab4d85b9eaef872d080faf_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:17,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/cde697ecff374d90a48e7d617a3e8cd8, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:17,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/cde697ecff374d90a48e7d617a3e8cd8 is 175, key is test_row_0/A:col10/1731768856030/Put/seqid=0 2024-11-16T14:54:17,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742425_1601 (size=31255) 2024-11-16T14:54:18,084 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/bcf5247e1b194167a14f90f85a835d46 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bcf5247e1b194167a14f90f85a835d46 2024-11-16T14:54:18,089 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/B of ca54beef0657cc499b531409f1c5b137 into bcf5247e1b194167a14f90f85a835d46(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:18,089 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:18,089 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/B, priority=13, startTime=1731768857669; duration=0sec 2024-11-16T14:54:18,089 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T14:54:18,089 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:B 2024-11-16T14:54:18,089 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T14:54:18,090 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T14:54:18,090 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1541): ca54beef0657cc499b531409f1c5b137/C is initiating minor compaction (all files) 2024-11-16T14:54:18,090 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca54beef0657cc499b531409f1c5b137/C in TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:18,090 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bbb970fcd6cd4158931ab6cef60f6ace, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/aaa7b0a9fd68498488342bad1f248551, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9f1b035b39e54220a53aee129d21830d] into tmpdir=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp, totalSize=36.4 K 2024-11-16T14:54:18,090 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting bbb970fcd6cd4158931ab6cef60f6ace, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731768854123 2024-11-16T14:54:18,091 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting aaa7b0a9fd68498488342bad1f248551, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731768854200 2024-11-16T14:54:18,091 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f1b035b39e54220a53aee129d21830d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731768855365 2024-11-16T14:54:18,099 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#C#compaction#507 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:18,100 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/734236b2a7b34cbd87890e09394499c2 is 50, key is test_row_0/C:col10/1731768856003/Put/seqid=0 2024-11-16T14:54:18,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742426_1602 (size=12949) 2024-11-16T14:54:18,113 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca54beef0657cc499b531409f1c5b137#A#compaction#505 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T14:54:18,114 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/9db79ee8fec749d391d7448c6b4bca72 is 175, key is test_row_0/A:col10/1731768856003/Put/seqid=0 2024-11-16T14:54:18,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742427_1603 (size=31903) 2024-11-16T14:54:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43955 {}] regionserver.HRegion(8855): Flush requested on ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:18,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. as already flushing 2024-11-16T14:54:18,183 INFO [Thread-2343 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2343. 2024-11-16T14:54:18,183 INFO [Thread-2345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2345. 2024-11-16T14:54:18,183 DEBUG [Thread-2345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:54:18,183 DEBUG [Thread-2343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:54:18,183 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:18,183 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:18,183 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:18,183 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:18,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:18,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:18,192 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-11-16T14:54:18,192 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:54:18,192 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:18,192 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:18,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:18,194 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/cde697ecff374d90a48e7d617a3e8cd8 2024-11-16T14:54:18,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/bd4210ac0e414256a54bff4a0fa15d47 is 50, key is test_row_0/B:col10/1731768856030/Put/seqid=0 2024-11-16T14:54:18,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742428_1604 (size=12301) 2024-11-16T14:54:18,505 INFO [Thread-2347 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2347. 2024-11-16T14:54:18,505 DEBUG [Thread-2347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:54:18,506 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:18,506 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:18,506 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:18,512 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/734236b2a7b34cbd87890e09394499c2 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/734236b2a7b34cbd87890e09394499c2 2024-11-16T14:54:18,517 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/C of ca54beef0657cc499b531409f1c5b137 into 734236b2a7b34cbd87890e09394499c2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:18,517 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:18,517 INFO [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/C, priority=13, startTime=1731768857669; duration=0sec 2024-11-16T14:54:18,517 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:18,517 DEBUG [RS:0;fc3fda29f660:43955-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:C 2024-11-16T14:54:18,521 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/9db79ee8fec749d391d7448c6b4bca72 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/9db79ee8fec749d391d7448c6b4bca72 2024-11-16T14:54:18,525 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca54beef0657cc499b531409f1c5b137/A of ca54beef0657cc499b531409f1c5b137 into 9db79ee8fec749d391d7448c6b4bca72(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T14:54:18,525 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:18,525 INFO [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., storeName=ca54beef0657cc499b531409f1c5b137/A, priority=13, startTime=1731768857669; duration=0sec 2024-11-16T14:54:18,525 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T14:54:18,525 DEBUG [RS:0;fc3fda29f660:43955-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca54beef0657cc499b531409f1c5b137:A 2024-11-16T14:54:18,532 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-11-16T14:54:18,532 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T14:54:18,532 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:18,532 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:18,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:18,604 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/bd4210ac0e414256a54bff4a0fa15d47 2024-11-16T14:54:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/eb883e2554b34501a4c9ceeaccb8ddcd is 50, key is test_row_0/C:col10/1731768856030/Put/seqid=0 2024-11-16T14:54:18,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742429_1605 (size=12301) 2024-11-16T14:54:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T14:54:19,022 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/eb883e2554b34501a4c9ceeaccb8ddcd 2024-11-16T14:54:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/cde697ecff374d90a48e7d617a3e8cd8 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cde697ecff374d90a48e7d617a3e8cd8 2024-11-16T14:54:19,034 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cde697ecff374d90a48e7d617a3e8cd8, entries=150, sequenceid=301, filesize=30.5 K 2024-11-16T14:54:19,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/bd4210ac0e414256a54bff4a0fa15d47 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bd4210ac0e414256a54bff4a0fa15d47 2024-11-16T14:54:19,038 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bd4210ac0e414256a54bff4a0fa15d47, entries=150, sequenceid=301, filesize=12.0 K 2024-11-16T14:54:19,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/eb883e2554b34501a4c9ceeaccb8ddcd as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/eb883e2554b34501a4c9ceeaccb8ddcd 2024-11-16T14:54:19,043 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/eb883e2554b34501a4c9ceeaccb8ddcd, entries=150, sequenceid=301, filesize=12.0 K 2024-11-16T14:54:19,043 INFO [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=33.54 KB/34350 for ca54beef0657cc499b531409f1c5b137 in 1267ms, sequenceid=301, compaction requested=false 2024-11-16T14:54:19,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for ca54beef0657cc499b531409f1c5b137: 2024-11-16T14:54:19,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:19,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-16T14:54:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-16T14:54:19,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-16T14:54:19,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4920 sec 2024-11-16T14:54:19,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.4960 sec 2024-11-16T14:54:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T14:54:20,721 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3015 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3245 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2994 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3012 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3245 2024-11-16T14:54:20,721 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T14:54:20,722 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:54:20,722 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:54:20,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:20,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:20,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:20,722 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T14:54:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T14:54:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T14:54:20,725 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768860725"}]},"ts":"1731768860725"} 2024-11-16T14:54:20,726 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T14:54:20,726 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T14:54:20,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T14:54:20,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, UNASSIGN}] 2024-11-16T14:54:20,728 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, UNASSIGN 2024-11-16T14:54:20,729 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=ca54beef0657cc499b531409f1c5b137, regionState=CLOSING, regionLocation=fc3fda29f660,43955,1731768673386 2024-11-16T14:54:20,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, UNASSIGN because future has completed 2024-11-16T14:54:20,730 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T14:54:20,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386}] 2024-11-16T14:54:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T14:54:20,881 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:20,881 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T14:54:20,881 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing ca54beef0657cc499b531409f1c5b137, disabling compactions & flushes 2024-11-16T14:54:20,881 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:20,882 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:20,882 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. after waiting 0 ms 2024-11-16T14:54:20,882 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:20,882 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing ca54beef0657cc499b531409f1c5b137 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T14:54:20,882 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=A 2024-11-16T14:54:20,882 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:20,882 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=B 2024-11-16T14:54:20,883 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:20,883 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca54beef0657cc499b531409f1c5b137, store=C 2024-11-16T14:54:20,883 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T14:54:20,892 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116631ddbf361514266818db0ab6eb0388d_ca54beef0657cc499b531409f1c5b137 is 50, key is test_row_0/A:col10/1731768858502/Put/seqid=0 2024-11-16T14:54:20,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742430_1606 (size=12454) 2024-11-16T14:54:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T14:54:21,299 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:54:21,308 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116631ddbf361514266818db0ab6eb0388d_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116631ddbf361514266818db0ab6eb0388d_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:21,309 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/a1c0e280d3d14698911b47afb8da16a0, store: [table=TestAcidGuarantees family=A region=ca54beef0657cc499b531409f1c5b137] 2024-11-16T14:54:21,309 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/a1c0e280d3d14698911b47afb8da16a0 is 175, key is test_row_0/A:col10/1731768858502/Put/seqid=0 2024-11-16T14:54:21,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742431_1607 (size=31255) 2024-11-16T14:54:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T14:54:21,713 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/a1c0e280d3d14698911b47afb8da16a0 2024-11-16T14:54:21,725 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/97243434c45a4dc583c29c7e36091204 is 50, key is test_row_0/B:col10/1731768858502/Put/seqid=0 2024-11-16T14:54:21,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742432_1608 (size=12301) 2024-11-16T14:54:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T14:54:22,132 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/97243434c45a4dc583c29c7e36091204 2024-11-16T14:54:22,140 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/331e6afa7ce049b5a047191ba0228111 is 50, key is test_row_0/C:col10/1731768858502/Put/seqid=0 2024-11-16T14:54:22,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742433_1609 (size=12301) 2024-11-16T14:54:22,545 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/331e6afa7ce049b5a047191ba0228111 2024-11-16T14:54:22,550 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/A/a1c0e280d3d14698911b47afb8da16a0 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/a1c0e280d3d14698911b47afb8da16a0 2024-11-16T14:54:22,554 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/a1c0e280d3d14698911b47afb8da16a0, entries=150, sequenceid=312, filesize=30.5 K 2024-11-16T14:54:22,555 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/B/97243434c45a4dc583c29c7e36091204 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/97243434c45a4dc583c29c7e36091204 2024-11-16T14:54:22,559 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/97243434c45a4dc583c29c7e36091204, entries=150, sequenceid=312, filesize=12.0 K 2024-11-16T14:54:22,560 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/.tmp/C/331e6afa7ce049b5a047191ba0228111 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/331e6afa7ce049b5a047191ba0228111 2024-11-16T14:54:22,564 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/331e6afa7ce049b5a047191ba0228111, entries=150, sequenceid=312, filesize=12.0 K 2024-11-16T14:54:22,564 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ca54beef0657cc499b531409f1c5b137 in 1682ms, sequenceid=312, compaction requested=true 2024-11-16T14:54:22,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/48be10c9dfca48e1a75773195d5a4ea1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/793cb98dced340a9ae12d83e14d58520, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1b6d05a7e73347d8b908186ef4d405d3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/e2ce2fe938ec485a9a8cc606d645b65f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cf3571497f474bf4b67cde8882e23b98, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/aad7d1eddedb4b93b04e76f45bb2d417, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/9903be8b525b4b809d12f0066fec6997, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/70c40893ad20458e919f898c0c88868b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1526f7f8e24546a485fe59b69141297d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/905f6ec1da564cf9a3e44079efe2aea9, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/25bd27bfacf045bbae7c882f738c4d8b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/36452d8720ca4b2e93e2cc4beb0609d6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/ba347b6085cf4a59a20eff09add0355e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/eb92f47b6f9b43b8b19ede0e90d7123d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/c987b9d772da47838caf7cbfc36f76c5, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/78237100fc5241e29487355de0e95e5b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/4b1e94ff751e4988995ae311f1d7ff09, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/8a7d566e1fae4492bf7d78648825e7f3, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/d063e414852145249e2c4353d8488b34] to archive 2024-11-16T14:54:22,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:54:22,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/48be10c9dfca48e1a75773195d5a4ea1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/48be10c9dfca48e1a75773195d5a4ea1 2024-11-16T14:54:22,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/793cb98dced340a9ae12d83e14d58520 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/793cb98dced340a9ae12d83e14d58520 2024-11-16T14:54:22,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1b6d05a7e73347d8b908186ef4d405d3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1b6d05a7e73347d8b908186ef4d405d3 2024-11-16T14:54:22,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/e2ce2fe938ec485a9a8cc606d645b65f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/e2ce2fe938ec485a9a8cc606d645b65f 2024-11-16T14:54:22,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cf3571497f474bf4b67cde8882e23b98 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cf3571497f474bf4b67cde8882e23b98 2024-11-16T14:54:22,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/aad7d1eddedb4b93b04e76f45bb2d417 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/aad7d1eddedb4b93b04e76f45bb2d417 2024-11-16T14:54:22,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/9903be8b525b4b809d12f0066fec6997 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/9903be8b525b4b809d12f0066fec6997 2024-11-16T14:54:22,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/70c40893ad20458e919f898c0c88868b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/70c40893ad20458e919f898c0c88868b 2024-11-16T14:54:22,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1526f7f8e24546a485fe59b69141297d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/1526f7f8e24546a485fe59b69141297d 2024-11-16T14:54:22,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/905f6ec1da564cf9a3e44079efe2aea9 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/905f6ec1da564cf9a3e44079efe2aea9 2024-11-16T14:54:22,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/25bd27bfacf045bbae7c882f738c4d8b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/25bd27bfacf045bbae7c882f738c4d8b 2024-11-16T14:54:22,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/36452d8720ca4b2e93e2cc4beb0609d6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/36452d8720ca4b2e93e2cc4beb0609d6 2024-11-16T14:54:22,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/ba347b6085cf4a59a20eff09add0355e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/ba347b6085cf4a59a20eff09add0355e 2024-11-16T14:54:22,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/eb92f47b6f9b43b8b19ede0e90d7123d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/eb92f47b6f9b43b8b19ede0e90d7123d 2024-11-16T14:54:22,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/c987b9d772da47838caf7cbfc36f76c5 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/c987b9d772da47838caf7cbfc36f76c5 2024-11-16T14:54:22,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/78237100fc5241e29487355de0e95e5b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/78237100fc5241e29487355de0e95e5b 2024-11-16T14:54:22,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/4b1e94ff751e4988995ae311f1d7ff09 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/4b1e94ff751e4988995ae311f1d7ff09 2024-11-16T14:54:22,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/8a7d566e1fae4492bf7d78648825e7f3 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/8a7d566e1fae4492bf7d78648825e7f3 2024-11-16T14:54:22,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/d063e414852145249e2c4353d8488b34 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/d063e414852145249e2c4353d8488b34 2024-11-16T14:54:22,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/ff86fcc5e3934b3ea8c8f8c7df77ab07, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/a553eebadb94465f93a934239d1b748c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/fa9f00c65f9a4c97ba50bcfbe080e078, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/8d5e9dade95942ce92ecc9e55c419771, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/5fc05db689a44df5bf490e8ed6b7b76b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/e5d36314103e45d79aaf40e4a8db9b11, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/893e4192bda54f46bc9ed4f587c2e3ba, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/2acaad00e91a452e9d0a643b750a5128, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bd024b506d5640a6907890f66e733eb7, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/04838ec115c44189848ecdb4c42322f1, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/9d0f20cf1e30477db3f74733950204e8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d1d5bad216cd42d8b509ba61a735a7f6, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/f3fe7227f16640edb86c3fa0edbe529b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/6b12856e39fc4da18ace378e07ce9e49, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d7a86a8f0dbf49dd99ed796eb2c6f28a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/43ffcdad59c44fc0b2dfb020e2e484cc, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/aa55dd077a354e7389196880e0178988, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d47a6f98b01b4ebfab82d2a254825105, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d24b01036ee84f0ab0c275254256da20] to archive 2024-11-16T14:54:22,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:54:22,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/ff86fcc5e3934b3ea8c8f8c7df77ab07 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/ff86fcc5e3934b3ea8c8f8c7df77ab07 2024-11-16T14:54:22,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/a553eebadb94465f93a934239d1b748c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/a553eebadb94465f93a934239d1b748c 2024-11-16T14:54:22,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/fa9f00c65f9a4c97ba50bcfbe080e078 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/fa9f00c65f9a4c97ba50bcfbe080e078 2024-11-16T14:54:22,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/8d5e9dade95942ce92ecc9e55c419771 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/8d5e9dade95942ce92ecc9e55c419771 2024-11-16T14:54:22,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/5fc05db689a44df5bf490e8ed6b7b76b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/5fc05db689a44df5bf490e8ed6b7b76b 2024-11-16T14:54:22,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/e5d36314103e45d79aaf40e4a8db9b11 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/e5d36314103e45d79aaf40e4a8db9b11 2024-11-16T14:54:22,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/893e4192bda54f46bc9ed4f587c2e3ba to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/893e4192bda54f46bc9ed4f587c2e3ba 2024-11-16T14:54:22,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/2acaad00e91a452e9d0a643b750a5128 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/2acaad00e91a452e9d0a643b750a5128 2024-11-16T14:54:22,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bd024b506d5640a6907890f66e733eb7 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bd024b506d5640a6907890f66e733eb7 2024-11-16T14:54:22,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/04838ec115c44189848ecdb4c42322f1 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/04838ec115c44189848ecdb4c42322f1 2024-11-16T14:54:22,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/9d0f20cf1e30477db3f74733950204e8 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/9d0f20cf1e30477db3f74733950204e8 2024-11-16T14:54:22,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d1d5bad216cd42d8b509ba61a735a7f6 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d1d5bad216cd42d8b509ba61a735a7f6 2024-11-16T14:54:22,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/f3fe7227f16640edb86c3fa0edbe529b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/f3fe7227f16640edb86c3fa0edbe529b 2024-11-16T14:54:22,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/6b12856e39fc4da18ace378e07ce9e49 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/6b12856e39fc4da18ace378e07ce9e49 2024-11-16T14:54:22,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d7a86a8f0dbf49dd99ed796eb2c6f28a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d7a86a8f0dbf49dd99ed796eb2c6f28a 2024-11-16T14:54:22,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/43ffcdad59c44fc0b2dfb020e2e484cc to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/43ffcdad59c44fc0b2dfb020e2e484cc 2024-11-16T14:54:22,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/aa55dd077a354e7389196880e0178988 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/aa55dd077a354e7389196880e0178988 2024-11-16T14:54:22,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d47a6f98b01b4ebfab82d2a254825105 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d47a6f98b01b4ebfab82d2a254825105 2024-11-16T14:54:22,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d24b01036ee84f0ab0c275254256da20 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/d24b01036ee84f0ab0c275254256da20 2024-11-16T14:54:22,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/1fbfd2cf6f6b4e469cc36539fc1a1b36, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f2046c1a07404f059ed725f26f41c295, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/3da308a9f5ed4dceb22c32ccb34fb527, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/a0e6e7a1a73c41db84e74b9433469d3a, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/2b4ec4befc094fe494d3c252b727f199, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/ebe4bde8199a45ccab04624eb32647f8, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/66bf75b469f3479b9c75afdb2118a03d, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/547785effa294d9eb6454b1b8097f98c, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/e166488b64184703870105b99ad62e8b, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f646b41863054adeb286847dea7fca14, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/6617c769707e44808974ac4b981d04a4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bf7231bf2dfe4ad7bdf8b6eb9094d189, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9283178426e34a7e9f0317f30ff6255e, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/92c953f8f1ca4ae093176bbc7c41e42f, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/c5ab065eade94112b46dad0210349616, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bbb970fcd6cd4158931ab6cef60f6ace, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/4ac02d5fa3b64242bfd0c97c540f20e4, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/aaa7b0a9fd68498488342bad1f248551, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9f1b035b39e54220a53aee129d21830d] to archive 2024-11-16T14:54:22,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T14:54:22,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/1fbfd2cf6f6b4e469cc36539fc1a1b36 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/1fbfd2cf6f6b4e469cc36539fc1a1b36 2024-11-16T14:54:22,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f2046c1a07404f059ed725f26f41c295 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f2046c1a07404f059ed725f26f41c295 2024-11-16T14:54:22,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/3da308a9f5ed4dceb22c32ccb34fb527 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/3da308a9f5ed4dceb22c32ccb34fb527 2024-11-16T14:54:22,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/a0e6e7a1a73c41db84e74b9433469d3a to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/a0e6e7a1a73c41db84e74b9433469d3a 2024-11-16T14:54:22,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/2b4ec4befc094fe494d3c252b727f199 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/2b4ec4befc094fe494d3c252b727f199 2024-11-16T14:54:22,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/ebe4bde8199a45ccab04624eb32647f8 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/ebe4bde8199a45ccab04624eb32647f8 2024-11-16T14:54:22,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/66bf75b469f3479b9c75afdb2118a03d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/66bf75b469f3479b9c75afdb2118a03d 2024-11-16T14:54:22,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/547785effa294d9eb6454b1b8097f98c to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/547785effa294d9eb6454b1b8097f98c 2024-11-16T14:54:22,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/e166488b64184703870105b99ad62e8b to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/e166488b64184703870105b99ad62e8b 2024-11-16T14:54:22,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f646b41863054adeb286847dea7fca14 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/f646b41863054adeb286847dea7fca14 2024-11-16T14:54:22,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/6617c769707e44808974ac4b981d04a4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/6617c769707e44808974ac4b981d04a4 2024-11-16T14:54:22,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bf7231bf2dfe4ad7bdf8b6eb9094d189 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bf7231bf2dfe4ad7bdf8b6eb9094d189 2024-11-16T14:54:22,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9283178426e34a7e9f0317f30ff6255e to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9283178426e34a7e9f0317f30ff6255e 2024-11-16T14:54:22,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/92c953f8f1ca4ae093176bbc7c41e42f to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/92c953f8f1ca4ae093176bbc7c41e42f 2024-11-16T14:54:22,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/c5ab065eade94112b46dad0210349616 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/c5ab065eade94112b46dad0210349616 2024-11-16T14:54:22,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bbb970fcd6cd4158931ab6cef60f6ace to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/bbb970fcd6cd4158931ab6cef60f6ace 2024-11-16T14:54:22,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/4ac02d5fa3b64242bfd0c97c540f20e4 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/4ac02d5fa3b64242bfd0c97c540f20e4 2024-11-16T14:54:22,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/aaa7b0a9fd68498488342bad1f248551 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/aaa7b0a9fd68498488342bad1f248551 2024-11-16T14:54:22,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9f1b035b39e54220a53aee129d21830d to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/9f1b035b39e54220a53aee129d21830d 2024-11-16T14:54:22,613 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/recovered.edits/315.seqid, newMaxSeqId=315, maxSeqId=4 2024-11-16T14:54:22,614 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. 2024-11-16T14:54:22,614 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for ca54beef0657cc499b531409f1c5b137: Waiting for close lock at 1731768860881Running coprocessor pre-close hooks at 1731768860881Disabling compacts and flushes for region at 1731768860881Disabling writes for close at 1731768860882 (+1 ms)Obtaining lock to block concurrent updates at 1731768860882Preparing flush snapshotting stores in ca54beef0657cc499b531409f1c5b137 at 1731768860882Finished memstore snapshotting TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731768860883 (+1 ms)Flushing stores of TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137. at 1731768860884 (+1 ms)Flushing ca54beef0657cc499b531409f1c5b137/A: creating writer at 1731768860884Flushing mob file ca54beef0657cc499b531409f1c5b137/A: appending metadata at 1731768860892 (+8 ms)Flushing mob file ca54beef0657cc499b531409f1c5b137/A: closing flushed file at 1731768860892Flushing ca54beef0657cc499b531409f1c5b137/A: appending metadata at 1731768861309 (+417 ms)Flushing ca54beef0657cc499b531409f1c5b137/A: closing flushed file at 1731768861309Flushing ca54beef0657cc499b531409f1c5b137/B: creating writer at 1731768861721 (+412 ms)Flushing ca54beef0657cc499b531409f1c5b137/B: appending metadata at 1731768861725 (+4 ms)Flushing ca54beef0657cc499b531409f1c5b137/B: closing flushed file at 1731768861725Flushing ca54beef0657cc499b531409f1c5b137/C: creating writer at 1731768862136 (+411 ms)Flushing ca54beef0657cc499b531409f1c5b137/C: appending metadata at 1731768862140 (+4 ms)Flushing ca54beef0657cc499b531409f1c5b137/C: closing flushed file at 1731768862140Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14d7033e: reopening flushed file at 1731768862549 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4786fd9: reopening flushed file at 1731768862554 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@701482bc: reopening flushed file at 1731768862559 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ca54beef0657cc499b531409f1c5b137 in 1682ms, sequenceid=312, compaction requested=true at 1731768862564 (+5 ms)Writing region close event to WAL at 1731768862611 (+47 ms)Running coprocessor post-close hooks at 1731768862614 (+3 ms)Closed at 1731768862614 2024-11-16T14:54:22,615 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,615 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=ca54beef0657cc499b531409f1c5b137, regionState=CLOSED 2024-11-16T14:54:22,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 because future has completed 2024-11-16T14:54:22,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-16T14:54:22,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure ca54beef0657cc499b531409f1c5b137, server=fc3fda29f660,43955,1731768673386 in 1.8870 sec 2024-11-16T14:54:22,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-16T14:54:22,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca54beef0657cc499b531409f1c5b137, UNASSIGN in 1.8900 sec 2024-11-16T14:54:22,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-16T14:54:22,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8920 sec 2024-11-16T14:54:22,621 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731768862621"}]},"ts":"1731768862621"} 2024-11-16T14:54:22,622 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T14:54:22,622 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T14:54:22,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9000 sec 2024-11-16T14:54:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T14:54:22,872 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:54:22,873 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T14:54:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:54:22,876 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:54:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T14:54:22,878 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:54:22,882 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,885 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C, FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/recovered.edits] 2024-11-16T14:54:22,889 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/9db79ee8fec749d391d7448c6b4bca72 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/9db79ee8fec749d391d7448c6b4bca72 2024-11-16T14:54:22,891 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/a1c0e280d3d14698911b47afb8da16a0 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/a1c0e280d3d14698911b47afb8da16a0 2024-11-16T14:54:22,893 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cde697ecff374d90a48e7d617a3e8cd8 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/A/cde697ecff374d90a48e7d617a3e8cd8 2024-11-16T14:54:22,898 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/97243434c45a4dc583c29c7e36091204 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/97243434c45a4dc583c29c7e36091204 2024-11-16T14:54:22,900 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bcf5247e1b194167a14f90f85a835d46 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bcf5247e1b194167a14f90f85a835d46 2024-11-16T14:54:22,902 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bd4210ac0e414256a54bff4a0fa15d47 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/B/bd4210ac0e414256a54bff4a0fa15d47 2024-11-16T14:54:22,905 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/331e6afa7ce049b5a047191ba0228111 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/331e6afa7ce049b5a047191ba0228111 2024-11-16T14:54:22,907 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/734236b2a7b34cbd87890e09394499c2 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/734236b2a7b34cbd87890e09394499c2 2024-11-16T14:54:22,908 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/eb883e2554b34501a4c9ceeaccb8ddcd to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/C/eb883e2554b34501a4c9ceeaccb8ddcd 2024-11-16T14:54:22,912 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/recovered.edits/315.seqid to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137/recovered.edits/315.seqid 2024-11-16T14:54:22,913 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/default/TestAcidGuarantees/ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,913 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T14:54:22,913 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:54:22,915 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T14:54:22,919 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162538c56d14f443d9842528b7aa9411f9_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162538c56d14f443d9842528b7aa9411f9_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,920 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162a8976d871c34b039ee189e7da2d4097_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162a8976d871c34b039ee189e7da2d4097_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,921 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111638bc699a4a2f4d3b8a97cd934ba08294_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111638bc699a4a2f4d3b8a97cd934ba08294_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,923 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116482de9229c7a4440814dcb2e6377237a_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116482de9229c7a4440814dcb2e6377237a_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,924 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165fefa040b02d4221bfec337af3417473_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165fefa040b02d4221bfec337af3417473_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,925 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166270198b20a443898038c04ccd573ecf_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166270198b20a443898038c04ccd573ecf_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,926 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116631ddbf361514266818db0ab6eb0388d_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116631ddbf361514266818db0ab6eb0388d_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,928 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166d29f62b214f4ab2a92a99ffa4a023ca_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166d29f62b214f4ab2a92a99ffa4a023ca_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,929 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116700859ed8fc44578b4f0b49d7cefeefa_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116700859ed8fc44578b4f0b49d7cefeefa_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,930 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a8aac3337c0048ba95fbe108892d9aae_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a8aac3337c0048ba95fbe108892d9aae_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,932 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a8add5867b274f14bbf1f414213d840c_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a8add5867b274f14bbf1f414213d840c_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,933 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e5744c535bb14f769fb821323719c875_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e5744c535bb14f769fb821323719c875_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,935 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e5b549da60944cf195b407224b741795_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e5b549da60944cf195b407224b741795_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,935 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e7e1fe841e1f4966b0d9ca5eeaca6e13_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e7e1fe841e1f4966b0d9ca5eeaca6e13_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,936 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ec484a1abeab4d85b9eaef872d080faf_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ec484a1abeab4d85b9eaef872d080faf_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,937 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fccd101ee57b41e099516ce62e13b624_ca54beef0657cc499b531409f1c5b137 to hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fccd101ee57b41e099516ce62e13b624_ca54beef0657cc499b531409f1c5b137 2024-11-16T14:54:22,937 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T14:54:22,939 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:54:22,941 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T14:54:22,942 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T14:54:22,943 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:54:22,943 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T14:54:22,943 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731768862943"}]},"ts":"9223372036854775807"} 2024-11-16T14:54:22,944 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T14:54:22,944 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ca54beef0657cc499b531409f1c5b137, NAME => 'TestAcidGuarantees,,1731768834315.ca54beef0657cc499b531409f1c5b137.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T14:54:22,944 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T14:54:22,945 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731768862944"}]},"ts":"9223372036854775807"} 2024-11-16T14:54:22,946 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T14:54:22,946 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T14:54:22,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-16T14:54:22,962 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T14:54:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T14:54:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T14:54:22,981 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T14:54:22,996 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252 (was 246) - Thread LEAK? -, OpenFileDescriptor=457 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=416 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=5295 (was 5480) 2024-11-16T14:54:22,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T14:54:22,997 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:54:22,997 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:54:22,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:22,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:22,997 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T14:54:22,997 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T14:54:22,997 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=387043539, stopped=false 2024-11-16T14:54:22,998 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fc3fda29f660,35823,1731768672605 2024-11-16T14:54:23,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T14:54:23,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T14:54:23,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:54:23,058 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:54:23,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:54:23,059 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T14:54:23,060 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:54:23,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:23,060 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:54:23,060 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:54:23,061 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fc3fda29f660,43955,1731768673386' ***** 2024-11-16T14:54:23,061 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T14:54:23,061 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T14:54:23,062 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T14:54:23,062 INFO [RS:0;fc3fda29f660:43955 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T14:54:23,062 INFO [RS:0;fc3fda29f660:43955 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T14:54:23,062 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(959): stopping server fc3fda29f660,43955,1731768673386 2024-11-16T14:54:23,062 INFO [RS:0;fc3fda29f660:43955 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T14:54:23,062 INFO [RS:0;fc3fda29f660:43955 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fc3fda29f660:43955. 2024-11-16T14:54:23,062 DEBUG [RS:0;fc3fda29f660:43955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:54:23,063 DEBUG [RS:0;fc3fda29f660:43955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:23,063 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T14:54:23,063 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T14:54:23,063 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T14:54:23,063 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T14:54:23,064 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T14:54:23,064 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T14:54:23,064 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T14:54:23,064 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T14:54:23,064 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T14:54:23,064 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T14:54:23,064 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T14:54:23,064 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:54:23,065 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-16T14:54:23,084 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/info/7eec69b7e84d4201949a36f43032e475 is 95, key is TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9./info:/1731768703650/DeleteFamily/seqid=0 2024-11-16T14:54:23,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742434_1610 (size=6514) 2024-11-16T14:54:23,157 INFO [regionserver/fc3fda29f660:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:54:23,265 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:54:23,447 INFO [regionserver/fc3fda29f660:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T14:54:23,447 INFO [regionserver/fc3fda29f660:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T14:54:23,465 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:54:23,488 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/info/7eec69b7e84d4201949a36f43032e475 2024-11-16T14:54:23,515 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/ns/5acfe9ec479443aea61d59da8c3a4174 is 93, key is TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9./ns:/1731768703610/DeleteFamily/seqid=0 2024-11-16T14:54:23,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742435_1611 (size=5906) 2024-11-16T14:54:23,666 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:54:23,866 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:54:23,918 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/ns/5acfe9ec479443aea61d59da8c3a4174 2024-11-16T14:54:23,940 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/rep_barrier/8a38bd4834654999b6d540ecc4f2b7c3 is 102, key is TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9./rep_barrier:/1731768703610/DeleteFamily/seqid=0 2024-11-16T14:54:23,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742436_1612 (size=6025) 2024-11-16T14:54:24,066 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:54:24,266 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:54:24,345 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/rep_barrier/8a38bd4834654999b6d540ecc4f2b7c3 2024-11-16T14:54:24,369 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/table/fe83907858794939a11053f2bf11a68d is 96, key is TestAcidGuarantees,,1731768677578.c410da0e7ea4faea21a8b41f311130b9./table:/1731768703610/DeleteFamily/seqid=0 2024-11-16T14:54:24,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742437_1613 (size=5936) 2024-11-16T14:54:24,467 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:54:24,667 DEBUG [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T14:54:24,773 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/table/fe83907858794939a11053f2bf11a68d 2024-11-16T14:54:24,783 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/info/7eec69b7e84d4201949a36f43032e475 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/info/7eec69b7e84d4201949a36f43032e475 2024-11-16T14:54:24,790 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/info/7eec69b7e84d4201949a36f43032e475, entries=12, sequenceid=90, filesize=6.4 K 2024-11-16T14:54:24,791 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/ns/5acfe9ec479443aea61d59da8c3a4174 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/ns/5acfe9ec479443aea61d59da8c3a4174 2024-11-16T14:54:24,795 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/ns/5acfe9ec479443aea61d59da8c3a4174, entries=8, sequenceid=90, filesize=5.8 K 2024-11-16T14:54:24,795 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/rep_barrier/8a38bd4834654999b6d540ecc4f2b7c3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/rep_barrier/8a38bd4834654999b6d540ecc4f2b7c3 2024-11-16T14:54:24,799 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/rep_barrier/8a38bd4834654999b6d540ecc4f2b7c3, entries=6, sequenceid=90, filesize=5.9 K 2024-11-16T14:54:24,799 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/.tmp/table/fe83907858794939a11053f2bf11a68d as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/table/fe83907858794939a11053f2bf11a68d 2024-11-16T14:54:24,802 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/table/fe83907858794939a11053f2bf11a68d, entries=7, sequenceid=90, filesize=5.8 K 2024-11-16T14:54:24,803 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1739ms, sequenceid=90, compaction requested=false 2024-11-16T14:54:24,808 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-16T14:54:24,809 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T14:54:24,809 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T14:54:24,809 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731768863064Running coprocessor pre-close hooks at 1731768863064Disabling compacts and flushes for region at 1731768863064Disabling writes for close at 1731768863064Obtaining lock to block concurrent updates at 1731768863065 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731768863065Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731768863065Flushing stores of hbase:meta,,1.1588230740 at 1731768863066 (+1 ms)Flushing 1588230740/info: creating writer at 1731768863066Flushing 1588230740/info: appending metadata at 1731768863084 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731768863084Flushing 1588230740/ns: creating writer at 1731768863498 (+414 ms)Flushing 1588230740/ns: appending metadata at 1731768863515 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731768863515Flushing 1588230740/rep_barrier: creating writer at 1731768863923 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731768863940 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731768863940Flushing 1588230740/table: creating writer at 1731768864352 (+412 ms)Flushing 1588230740/table: appending metadata at 1731768864369 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731768864369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@235466ae: reopening flushed file at 1731768864781 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11198e8e: reopening flushed file at 1731768864790 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a60cbaf: reopening flushed file at 1731768864795 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37e43207: reopening flushed file at 1731768864799 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1739ms, sequenceid=90, compaction requested=false at 1731768864803 (+4 ms)Writing region close event to WAL at 1731768864805 (+2 ms)Running coprocessor post-close hooks at 1731768864808 (+3 ms)Closed at 1731768864809 (+1 ms) 2024-11-16T14:54:24,809 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T14:54:24,867 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(976): stopping server fc3fda29f660,43955,1731768673386; all regions closed. 2024-11-16T14:54:24,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741834_1010 (size=25344) 2024-11-16T14:54:24,881 DEBUG [RS:0;fc3fda29f660:43955 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/oldWALs 2024-11-16T14:54:24,881 INFO [RS:0;fc3fda29f660:43955 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fc3fda29f660%2C43955%2C1731768673386.meta:.meta(num 1731768677102) 2024-11-16T14:54:24,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741833_1009 (size=18471030) 2024-11-16T14:54:24,887 DEBUG [RS:0;fc3fda29f660:43955 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/oldWALs 2024-11-16T14:54:24,887 INFO [RS:0;fc3fda29f660:43955 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fc3fda29f660%2C43955%2C1731768673386:(num 1731768676576) 2024-11-16T14:54:24,887 DEBUG [RS:0;fc3fda29f660:43955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:54:24,887 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:54:24,887 INFO [RS:0;fc3fda29f660:43955 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T14:54:24,887 INFO [RS:0;fc3fda29f660:43955 {}] hbase.ChoreService(370): Chore service for: regionserver/fc3fda29f660:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T14:54:24,887 INFO [RS:0;fc3fda29f660:43955 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T14:54:24,887 INFO [regionserver/fc3fda29f660:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T14:54:24,888 INFO [RS:0;fc3fda29f660:43955 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43955 2024-11-16T14:54:24,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T14:54:24,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fc3fda29f660,43955,1731768673386 2024-11-16T14:54:24,940 INFO [RS:0;fc3fda29f660:43955 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T14:54:24,948 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fc3fda29f660,43955,1731768673386] 2024-11-16T14:54:24,956 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fc3fda29f660,43955,1731768673386 already deleted, retry=false 2024-11-16T14:54:24,956 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fc3fda29f660,43955,1731768673386 expired; onlineServers=0 2024-11-16T14:54:24,957 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fc3fda29f660,35823,1731768672605' ***** 2024-11-16T14:54:24,957 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T14:54:24,957 INFO [M:0;fc3fda29f660:35823 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T14:54:24,957 INFO [M:0;fc3fda29f660:35823 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T14:54:24,957 DEBUG [M:0;fc3fda29f660:35823 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T14:54:24,957 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T14:54:24,957 DEBUG [M:0;fc3fda29f660:35823 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T14:54:24,957 DEBUG [master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.small.0-1731768676178 {}] cleaner.HFileCleaner(306): Exit Thread[master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.small.0-1731768676178,5,FailOnTimeoutGroup] 2024-11-16T14:54:24,957 DEBUG [master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.large.0-1731768676176 {}] cleaner.HFileCleaner(306): Exit Thread[master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.large.0-1731768676176,5,FailOnTimeoutGroup] 2024-11-16T14:54:24,958 INFO [M:0;fc3fda29f660:35823 {}] hbase.ChoreService(370): Chore service for: master/fc3fda29f660:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T14:54:24,958 INFO [M:0;fc3fda29f660:35823 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T14:54:24,958 DEBUG [M:0;fc3fda29f660:35823 {}] master.HMaster(1795): Stopping service threads 2024-11-16T14:54:24,958 INFO [M:0;fc3fda29f660:35823 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T14:54:24,958 INFO [M:0;fc3fda29f660:35823 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:54:24,958 ERROR [M:0;fc3fda29f660:35823 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (879580466) connection to localhost/127.0.0.1:38773 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:38773,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-16T14:54:24,959 INFO [M:0;fc3fda29f660:35823 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T14:54:24,959 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T14:54:24,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T14:54:24,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:54:24,965 DEBUG [M:0;fc3fda29f660:35823 {}] zookeeper.ZKUtil(347): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T14:54:24,965 WARN [M:0;fc3fda29f660:35823 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T14:54:24,966 INFO [M:0;fc3fda29f660:35823 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/.lastflushedseqids 2024-11-16T14:54:24,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742438_1614 (size=72) 2024-11-16T14:54:25,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:54:25,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43955-0x101456a70e90001, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:54:25,050 INFO [RS:0;fc3fda29f660:43955 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T14:54:25,050 INFO [RS:0;fc3fda29f660:43955 {}] regionserver.HRegionServer(1031): Exiting; stopping=fc3fda29f660,43955,1731768673386; zookeeper connection closed. 2024-11-16T14:54:25,051 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@35228498 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@35228498 2024-11-16T14:54:25,052 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-16T14:54:25,379 INFO [M:0;fc3fda29f660:35823 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T14:54:25,380 INFO [M:0;fc3fda29f660:35823 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T14:54:25,380 DEBUG [M:0;fc3fda29f660:35823 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T14:54:25,380 INFO [M:0;fc3fda29f660:35823 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:54:25,380 DEBUG [M:0;fc3fda29f660:35823 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:54:25,380 DEBUG [M:0;fc3fda29f660:35823 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T14:54:25,380 DEBUG [M:0;fc3fda29f660:35823 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:54:25,380 INFO [M:0;fc3fda29f660:35823 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=779.12 KB heapSize=954.46 KB 2024-11-16T14:54:25,401 DEBUG [M:0;fc3fda29f660:35823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2049970d1b59417fad9175ed8bb74839 is 82, key is hbase:meta,,1/info:regioninfo/1731768677178/Put/seqid=0 2024-11-16T14:54:25,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742439_1615 (size=5672) 2024-11-16T14:54:25,805 INFO [M:0;fc3fda29f660:35823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2164 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2049970d1b59417fad9175ed8bb74839 2024-11-16T14:54:25,833 DEBUG [M:0;fc3fda29f660:35823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f412d132c559468da298a4bc6edbeeb3 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1731768837417/Put/seqid=0 2024-11-16T14:54:25,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742440_1616 (size=46127) 2024-11-16T14:54:26,238 INFO [M:0;fc3fda29f660:35823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=778.57 KB at sequenceid=2164 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f412d132c559468da298a4bc6edbeeb3 2024-11-16T14:54:26,245 INFO [M:0;fc3fda29f660:35823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f412d132c559468da298a4bc6edbeeb3 2024-11-16T14:54:26,265 DEBUG [M:0;fc3fda29f660:35823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/36359a70adf7421d94d34e76add6aba1 is 69, key is fc3fda29f660,43955,1731768673386/rs:state/1731768676319/Put/seqid=0 2024-11-16T14:54:26,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073742441_1617 (size=5156) 2024-11-16T14:54:26,669 INFO [M:0;fc3fda29f660:35823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2164 (bloomFilter=true), to=hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/36359a70adf7421d94d34e76add6aba1 2024-11-16T14:54:26,677 DEBUG [M:0;fc3fda29f660:35823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2049970d1b59417fad9175ed8bb74839 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2049970d1b59417fad9175ed8bb74839 2024-11-16T14:54:26,681 INFO [M:0;fc3fda29f660:35823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2049970d1b59417fad9175ed8bb74839, entries=8, sequenceid=2164, filesize=5.5 K 2024-11-16T14:54:26,682 DEBUG [M:0;fc3fda29f660:35823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f412d132c559468da298a4bc6edbeeb3 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f412d132c559468da298a4bc6edbeeb3 2024-11-16T14:54:26,685 INFO [M:0;fc3fda29f660:35823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f412d132c559468da298a4bc6edbeeb3 2024-11-16T14:54:26,685 INFO [M:0;fc3fda29f660:35823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f412d132c559468da298a4bc6edbeeb3, entries=174, sequenceid=2164, filesize=45.0 K 2024-11-16T14:54:26,685 DEBUG [M:0;fc3fda29f660:35823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/36359a70adf7421d94d34e76add6aba1 as hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/36359a70adf7421d94d34e76add6aba1 2024-11-16T14:54:26,688 INFO [M:0;fc3fda29f660:35823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38773/user/jenkins/test-data/5ac6f990-908f-e36b-82fe-d5fdb665dc7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/36359a70adf7421d94d34e76add6aba1, entries=1, sequenceid=2164, filesize=5.0 K 2024-11-16T14:54:26,689 INFO [M:0;fc3fda29f660:35823 {}] regionserver.HRegion(3140): Finished flush of dataSize ~779.12 KB/797820, heapSize ~954.16 KB/977064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2164, compaction requested=false 2024-11-16T14:54:26,690 INFO [M:0;fc3fda29f660:35823 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:54:26,690 DEBUG [M:0;fc3fda29f660:35823 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731768865380Disabling compacts and flushes for region at 1731768865380Disabling writes for close at 1731768865380Obtaining lock to block concurrent updates at 1731768865380Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731768865380Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=797820, getHeapSize=977304, getOffHeapSize=0, getCellsCount=2363 at 1731768865380Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731768865381 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731768865381Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731768865401 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731768865401Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731768865814 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731768865833 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731768865833Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731768866246 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731768866265 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731768866265Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d42c33e: reopening flushed file at 1731768866676 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8eb5b9f: reopening flushed file at 1731768866681 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@505db160: reopening flushed file at 1731768866685 (+4 ms)Finished flush of dataSize ~779.12 KB/797820, heapSize ~954.16 KB/977064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2164, compaction requested=false at 1731768866689 (+4 ms)Writing region close event to WAL at 1731768866690 (+1 ms)Closed at 1731768866690 2024-11-16T14:54:26,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741830_1006 (size=939695) 2024-11-16T14:54:26,693 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T14:54:26,693 INFO [M:0;fc3fda29f660:35823 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T14:54:26,693 INFO [M:0;fc3fda29f660:35823 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35823 2024-11-16T14:54:26,693 INFO [M:0;fc3fda29f660:35823 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T14:54:26,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:54:26,806 INFO [M:0;fc3fda29f660:35823 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T14:54:26,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35823-0x101456a70e90000, quorum=127.0.0.1:54092, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:54:26,818 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:54:26,820 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T14:54:26,821 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T14:54:26,821 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T14:54:26,821 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/hadoop.log.dir/,STOPPED} 2024-11-16T14:54:26,824 WARN [BP-443048117-172.17.0.2-1731768669338 heartbeating to localhost/127.0.0.1:38773 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T14:54:26,824 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T14:54:26,824 WARN [BP-443048117-172.17.0.2-1731768669338 heartbeating to localhost/127.0.0.1:38773 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-443048117-172.17.0.2-1731768669338 (Datanode Uuid 12639581-9049-4216-a9f7-1e0789fcc1d5) service to localhost/127.0.0.1:38773 2024-11-16T14:54:26,824 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T14:54:26,827 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/cluster_f34ac9e9-6867-0b4c-ab87-b17e81b33f1e/data/data1/current/BP-443048117-172.17.0.2-1731768669338 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T14:54:26,827 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/cluster_f34ac9e9-6867-0b4c-ab87-b17e81b33f1e/data/data2/current/BP-443048117-172.17.0.2-1731768669338 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T14:54:26,828 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T14:54:26,834 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T14:54:26,835 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T14:54:26,835 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T14:54:26,835 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T14:54:26,835 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/757399de-aa66-3a98-0af9-06ae64293954/hadoop.log.dir/,STOPPED} 2024-11-16T14:54:26,848 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T14:54:26,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down